use pest::Parser;
use pest_derive::Parser;

// 引入 pest 文法文件
#[derive(Parser)]
#[grammar = "lexer.pest"]
struct SysYLexer;

// 定义令牌类型
#[derive(Debug, Clone, PartialEq)]
pub enum Token {
    // 关键字
    Const,    // 'const'
    Int,      // 'int'
    Void,     // 'void'
    If,       // 'if'
    Else,     // 'else'
    While,    // 'while'
    Break,    // 'break'
    Continue, // 'continue'
    Return,   // 'return'
    // 运算符
    Plus,   // '+'
    Minus,  // '-'
    Mul,    // '*'
    Div,    // '/'
    Mod,    // '%'
    Assign, // '='
    Eq,     // '=='
    Neq,    // '!='
    Lt,     // '<'
    Gt,     // '>'
    Le,     // '<='
    Ge,     // '>='
    Not,    // '!'
    And,    // '&&'
    Or,     // '||'
    // 标点符号
    LParen,    // '('
    RParen,    // ')'
    LBrace,    // '{'
    RBrace,    // '}'
    LBracket,  // '['
    RBracket,  // ']'
    Comma,     // ','
    Semicolon, // ';'
    // 标识符和整数常量
    Ident(String),     // 标识符，如变量名
    IntegerConst(i64), // 整数常量（存储十进制值）
}

/// 将整数常量字符串转换为十进制值
fn parse_integer_const(s: &str) -> Result<i64, String> {
    let s = s.trim();
    
    if s.starts_with("0x") || s.starts_with("0X") {
        // 十六进制
        let hex_str = &s[2..];
        if hex_str.is_empty() {
            // 仅"0x"或"0X"视为0
            Ok(0)
        } else if hex_str.chars().all(|c| c.is_ascii_hexdigit()) {
            i64::from_str_radix(hex_str, 16)
                .map_err(|_| format!("Invalid hex number: {}", s))
        } else {
            // 包含非法十六进制字符，只取合法部分
            let valid_hex: String = hex_str.chars()
                .take_while(|c| c.is_ascii_hexdigit())
                .collect();
            if valid_hex.is_empty() {
                Ok(0)
            } else {
                i64::from_str_radix(&valid_hex, 16)
                    .map_err(|_| format!("Invalid hex number: {}", s))
            }
        }
    } else if s.starts_with('0') && s.len() > 1 {
        // 八进制或前导零的十进制
        let trimmed = s.trim_start_matches('0');
        if trimmed.is_empty() {
            Ok(0) // 仅0或多个0
        } else if trimmed.chars().all(|c| c >= '0' && c <= '7') && s.chars().take_while(|&c| c == '0').count() == 1 {
            // 严格八进制：单一前导0后跟0-7
            i64::from_str_radix(trimmed, 8)
                .map_err(|_| format!("Invalid octal number: {}", s))
        } else {
            // 视为十进制，包含非法八进制如"08"
            trimmed.parse::<i64>()
                .map_err(|_| format!("Invalid decimal number: {}", s))
        }
    } else {
        // 十进制
        s.parse::<i64>()
            .map_err(|_| format!("Invalid decimal number: {}", s))
    }
}

/// 预处理输入字符串，拆分不合法的数字序列
fn preprocess_numbers(input: &str) -> String {
    let mut result = String::with_capacity(input.len());
    let bytes = input.as_bytes();
    let mut i = 0;
    
    while i < bytes.len() {
        // 检查是否以数字开头
        if bytes[i] >= b'0' && bytes[i] <= b'9' {
            // 处理十六进制：0x 或 0X
            if i + 1 < bytes.len() && 
               bytes[i] == b'0' && 
               (bytes[i + 1] == b'x' || bytes[i + 1] == b'X') {
                
                result.push(bytes[i] as char); // '0'
                result.push(bytes[i + 1] as char); // 'x' 或 'X'
                i += 2;
                
                // 处理十六进制数字部分 - 第一个数字与0x组合，其余拆分
                if i < bytes.len() && 
                   ((bytes[i] >= b'0' && bytes[i] <= b'9') ||
                    (bytes[i] >= b'a' && bytes[i] <= b'f') ||
                    (bytes[i] >= b'A' && bytes[i] <= b'F')) {
                    // 第一个十六进制数字与0x组合
                    result.push(bytes[i] as char);
                    i += 1;
                    
                    // 后续数字拆分
                    while i < bytes.len() && 
                          ((bytes[i] >= b'0' && bytes[i] <= b'9') ||
                           (bytes[i] >= b'a' && bytes[i] <= b'f') ||
                           (bytes[i] >= b'A' && bytes[i] <= b'F')) {
                        result.push(' ');
                        result.push(bytes[i] as char);
                        i += 1;
                    }
                }
                continue;
            }
            
            // 处理以0开头的数字
            if bytes[i] == b'0' {
                let zero_start = i;
                while i < bytes.len() && bytes[i] == b'0' {
                    i += 1;
                }
                
                // 检查后面跟着什么
                if i < bytes.len() {
                    let next_char = bytes[i];
                    
                    // 如果后面是8或9，需要拆分
                    if next_char == b'8' || next_char == b'9' {
                        // 输出所有的 0
                        for _ in zero_start..i {
                            result.push('0');
                            result.push(' ');
                        }
                        // 输出 8 或 9
                        result.push(next_char as char);
                        i += 1;
                        continue;
                    }
                    
                    // 如果后面是字母或下划线，需要拆分
                    if (next_char >= b'a' && next_char <= b'z') ||
                       (next_char >= b'A' && next_char <= b'Z') ||
                       next_char == b'_' {
                        // 输出所有的 0
                        for _ in zero_start..i {
                            result.push('0');
                            result.push(' ');
                        }
                        continue;
                    }
                    
                    // 如果后面是0-7，检查是否应该拆分
                    if next_char >= b'0' && next_char <= b'7' {
                        // 检查是否有多于一个前导0
                        if i - zero_start > 1 {
                            // 多个前导0，需要拆分
                            for _ in zero_start..i {
                                result.push('0');
                                result.push(' ');
                            }
                            // 处理八进制数字部分
                            while i < bytes.len() && bytes[i] >= b'0' && bytes[i] <= b'7' {
                                result.push(bytes[i] as char);
                                i += 1;
                            }
                            continue;
                        } else {
                            // 只有一个前导0，保持完整
                            result.push_str(&input[zero_start..i]);
                            // 处理八进制数字部分
                            while i < bytes.len() && bytes[i] >= b'0' && bytes[i] <= b'7' {
                                result.push(bytes[i] as char);
                                i += 1;
                            }
                            continue;
                        }
                    }
                }
                
                // 其他情况，原样输出
                result.push_str(&input[zero_start..i]);
                continue;
            }
            
            // 处理十进制：1-9 开头
            if bytes[i] >= b'1' && bytes[i] <= b'9' {
                while i < bytes.len() && bytes[i] >= b'0' && bytes[i] <= b'9' {
                    result.push(bytes[i] as char);
                    i += 1;
                }
                continue;
            }
        }
        
        // 非数字字符，原样输出
        result.push(bytes[i] as char);
        i += 1;
    }
    
    result
}

/// 词法分析函数，将输入字符串解析为令牌序列
///
/// # 参数
/// - `input`: 输入的源代码字符串
///
/// # 返回值
/// - 成功时返回 `Vec<(Token, usize)>`，包含令牌及其行号
/// - 失败时返回错误信息，包含非法字符及其行号
pub fn tokenize(input: &str) -> Result<Vec<(Token, usize)>, (String, usize)> {
    // 1) 归一化换行：将 CRLF 和单独 CR 规范为 LF
    let mut normalized = input.replace("\r\n", "\n");
    normalized = normalized.replace('\r', "\n");
    
    // 2) 预处理数字拆分
    let preprocessed = preprocess_numbers(&normalized);
    
    let mut tokens = Vec::new();

    // 启用详细错误信息
    pest::set_error_detail(true);
    // 使用 SysYLexer 解析预处理后的字符串
    let pairs = match SysYLexer::parse(Rule::file, &preprocessed) {
        Ok(pairs) => pairs,
        Err(e) => {
            let message = format!("Mysterious character \"{}\".", e);
            let line_no = match e.line_col {
                pest::error::LineColLocation::Pos((line, _)) => line,
                pest::error::LineColLocation::Span((line, _), _) => line,
            };
            return Err((message, line_no));
        }
    };

    // 收集原始规则与位置信息，用于后处理 "NEQ" + "ASSIGN"（对应源码 "!=="）重写为 "NOT" + "EQ"
    #[derive(Clone, Copy)]
    struct TokSpan {
        rule: Rule,
        line: usize,
        start: usize,
        end: usize,
    }
    let mut raw: Vec<TokSpan> = Vec::new();

    for pair in pairs.flatten() {
        let span = pair.as_span();
        let line_no = span.start_pos().line_col().0;
        let start = span.start_pos().pos();
        let end = span.end_pos().pos();
        match pair.as_rule() {
            // 跳过空白和注释以及框架符号
            Rule::WHITESPACE | Rule::COMMENT | Rule::LINE_COMMENT | Rule::MULTILINE_COMMENT | Rule::file | Rule::EOI => {}
            // 记录其余 token 以便后续转换
            r => raw.push(TokSpan { rule: r, line: line_no, start, end }),
        }
    }

    // 后处理：将紧邻的 "NEQ" + "ASSIGN"（对应源码 "!=="）重写为 "NOT" + "EQ"
    let mut i = 0;
    while i < raw.len() {
        if i + 1 < raw.len()
            && raw[i].rule == Rule::NEQ
            && raw[i + 1].rule == Rule::ASSIGN
            && raw[i].end == raw[i + 1].start
            && raw[i].line == raw[i + 1].line
        {
            // 输出 NOT + EQ
            tokens.push((Token::Not, raw[i].line));
            tokens.push((Token::Eq, raw[i + 1].line));
            i += 2;
            continue;
        }

        // 常规映射
        let t = match raw[i].rule {
            // 关键字
            Rule::CONST => Token::Const,
            Rule::INT => Token::Int,
            Rule::VOID => Token::Void,
            Rule::IF => Token::If,
            Rule::ELSE => Token::Else,
            Rule::WHILE => Token::While,
            Rule::BREAK => Token::Break,
            Rule::CONTINUE => Token::Continue,
            Rule::RETURN => Token::Return,
            // 运算符
            Rule::PLUS => Token::Plus,
            Rule::MINUS => Token::Minus,
            Rule::MUL => Token::Mul,
            Rule::DIV => Token::Div,
            Rule::MOD => Token::Mod,
            Rule::ASSIGN => Token::Assign,
            Rule::EQ => Token::Eq,
            Rule::NEQ => Token::Neq,
            Rule::LT => Token::Lt,
            Rule::GT => Token::Gt,
            Rule::LE => Token::Le,
            Rule::GE => Token::Ge,
            Rule::NOT => Token::Not,
            Rule::AND => Token::And,
            Rule::OR => Token::Or,
            // 标点
            Rule::L_PAREN => Token::LParen,
            Rule::R_PAREN => Token::RParen,
            Rule::L_BRACE => Token::LBrace,
            Rule::R_BRACE => Token::RBrace,
            Rule::L_BRACKT => Token::LBracket,
            Rule::R_BRACKT => Token::RBracket,
            Rule::COMMA => Token::Comma,
            Rule::SEMICOLON => Token::Semicolon,
            // 标识符与整数：为了获取词素，需要从 preprocessed 切片
            Rule::IDENT => {
                // 恢复词素
                let lex = &preprocessed[raw[i].start..raw[i].end];
                Token::Ident(lex.to_string())
            }
            Rule::INTEGER_CONST => {
                let lex = &preprocessed[raw[i].start..raw[i].end];
                let value = parse_integer_const(lex).map_err(|msg| (msg, raw[i].line))?;
                Token::IntegerConst(value)
            }
            // 非法字符直接报错
            Rule::INVALID_CHAR => {
                let lex = &preprocessed[raw[i].start..raw[i].end];
                let message = format!("Mysterious character \"{}\".", lex);
                return Err((message, raw[i].line));
            }
            _ => { i += 1; continue; }
        };
        tokens.push((t, raw[i].line));
        i += 1;
    }

    Ok(tokens)
}

fn main() {
    // 收集命令行参数
    let args: Vec<String> = std::env::args().collect();

    // 检查是否提供了文件名
    if args.len() < 2 {
        eprintln!("Usage: {} <filename>", args[0]);
        std::process::exit(1);
    }

    // 获取文件名
    let filename = &args[1];

    // 读取输入文件
    let input = std::fs::read_to_string(filename).expect("Failed to read file");

    // 词法分析
    match tokenize(&input) {
        Ok(tokens) => {
            // 打印令牌
            for (token, line_no) in tokens {
                let (token_type, token_text) = match token {
                    Token::Const => ("CONST", "const".to_string()),
                    Token::Int => ("INT", "int".to_string()),
                    Token::Void => ("VOID", "void".to_string()),
                    Token::If => ("IF", "if".to_string()),
                    Token::Else => ("ELSE", "else".to_string()),
                    Token::While => ("WHILE", "while".to_string()),
                    Token::Break => ("BREAK", "break".to_string()),
                    Token::Continue => ("CONTINUE", "continue".to_string()),
                    Token::Return => ("RETURN", "return".to_string()),
                    Token::Plus => ("PLUS", "+".to_string()),
                    Token::Minus => ("MINUS", "-".to_string()),
                    Token::Mul => ("MUL", "*".to_string()),
                    Token::Div => ("DIV", "/".to_string()),
                    Token::Mod => ("MOD", "%".to_string()),
                    Token::Assign => ("ASSIGN", "=".to_string()),
                    Token::Eq => ("EQ", "==".to_string()),
                    Token::Neq => ("NEQ", "!=".to_string()),
                    Token::Lt => ("LT", "<".to_string()),
                    Token::Gt => ("GT", ">".to_string()),
                    Token::Le => ("LE", "<=".to_string()),
                    Token::Ge => ("GE", ">=".to_string()),
                    Token::Not => ("NOT", "!".to_string()),
                    Token::And => ("AND", "&&".to_string()),
                    Token::Or => ("OR", "||".to_string()),
                    Token::LParen => ("L_PAREN", "(".to_string()),
                    Token::RParen => ("R_PAREN", ")".to_string()),
                    Token::LBrace => ("L_BRACE", "{".to_string()),
                    Token::RBrace => ("R_BRACE", "}".to_string()),
                    Token::LBracket => ("L_BRACKT", "[".to_string()),
                    Token::RBracket => ("R_BRACKT", "]".to_string()),
                    Token::Comma => ("COMMA", ",".to_string()),
                    Token::Semicolon => ("SEMICOLON", ";".to_string()),
                    Token::Ident(s) => ("IDENT", s),
                    Token::IntegerConst(n) => ("INTEGER_CONST", n.to_string()),
                };
                eprintln!("{} {} at Line {}.", token_type, token_text, line_no);
            }
        }
        Err((msg, line_no)) => {
            eprintln!("Error type A at Line {}: {}", line_no, msg);
        }
    }
}

#[cfg(test)]
mod test {
    use super::*;

    // 辅助函数：将token转换为字符串格式
    fn token_to_string(token: &Token, line_no: usize) -> String {
        let (token_type, token_text) = match token {
            Token::Const => ("CONST", "const".to_string()),
            Token::Int => ("INT", "int".to_string()),
            Token::Void => ("VOID", "void".to_string()),
            Token::If => ("IF", "if".to_string()),
            Token::Else => ("ELSE", "else".to_string()),
            Token::While => ("WHILE", "while".to_string()),
            Token::Break => ("BREAK", "break".to_string()),
            Token::Continue => ("CONTINUE", "continue".to_string()),
            Token::Return => ("RETURN", "return".to_string()),
            Token::Plus => ("PLUS", "+".to_string()),
            Token::Minus => ("MINUS", "-".to_string()),
            Token::Mul => ("MUL", "*".to_string()),
            Token::Div => ("DIV", "/".to_string()),
            Token::Mod => ("MOD", "%".to_string()),
            Token::Assign => ("ASSIGN", "=".to_string()),
            Token::Eq => ("EQ", "==".to_string()),
            Token::Neq => ("NEQ", "!=".to_string()),
            Token::Lt => ("LT", "<".to_string()),
            Token::Gt => ("GT", ">".to_string()),
            Token::Le => ("LE", "<=".to_string()),
            Token::Ge => ("GE", ">=".to_string()),
            Token::Not => ("NOT", "!".to_string()),
            Token::And => ("AND", "&&".to_string()),
            Token::Or => ("OR", "||".to_string()),
            Token::LParen => ("L_PAREN", "(".to_string()),
            Token::RParen => ("R_PAREN", ")".to_string()),
            Token::LBrace => ("L_BRACE", "{".to_string()),
            Token::RBrace => ("R_BRACE", "}".to_string()),
            Token::LBracket => ("L_BRACKT", "[".to_string()),
            Token::RBracket => ("R_BRACKT", "]".to_string()),
            Token::Comma => ("COMMA", ",".to_string()),
            Token::Semicolon => ("SEMICOLON", ";".to_string()),
            Token::Ident(s) => ("IDENT", s.clone()),
            Token::IntegerConst(n) => ("INTEGER_CONST", n.to_string()),
        };
        format!("{} {} at Line {}.", token_type, token_text, line_no)
    }

    // 测试1: 样例一 - 基本函数定义与注释
    #[test]
    fn test_sample_1() {
        let input = r#"
int main()
{
   // line comment
   /*
     block comment
   */
   int i = 0x1;
}
"#;
        let result = tokenize(input);
        assert!(result.is_ok());
        
        let tokens = result.unwrap();
        let output: Vec<String> = tokens.iter()
            .map(|(token, line_no)| token_to_string(token, *line_no))
            .collect();
        
        assert_eq!(output, vec![
            "INT int at Line 2.".to_string(),
            "IDENT main at Line 2.".to_string(),
            "L_PAREN ( at Line 2.".to_string(),
            "R_PAREN ) at Line 2.".to_string(),
            "L_BRACE { at Line 3.".to_string(),
            "INT int at Line 8.".to_string(),
            "IDENT i at Line 8.".to_string(),
            "ASSIGN = at Line 8.".to_string(),
            "INTEGER_CONST 1 at Line 8.".to_string(),
            "SEMICOLON ; at Line 8.".to_string(),
            "R_BRACE } at Line 9.".to_string(),
        ]);
    }

    // 测试2: 样例二 - 非法字符
    #[test]
    fn test_sample_2() {
        let input = r#"
int main(){
  int i = 1;
  int j = ~i;
}
"#;
        let result = tokenize(input);
        assert!(result.is_err());
        
        let (msg, line_no) = result.unwrap_err();
        assert_eq!(msg, "Mysterious character \"~\".");
        assert_eq!(line_no, 4);
    }

    // 测试3: 样例三 - 复杂程序
    #[test]
    fn test_sample_3() {
        let input = r#"
int func(int arg) {
    int l;
    l = - - - arg;
    return l;
}

int main() {
    int x, y;
    x = 02;
    y = 0x1;
    x = x - 1 + y;
    if (+-!!!x) {
        x = - - -2;
    }
    else {
        x = 1 + + y;
    }
    func(x);
    return 0;
}
"#;
        let result = tokenize(input);
        assert!(result.is_ok());
        
        let tokens = result.unwrap();
        let output: Vec<String> = tokens.iter()
            .map(|(token, line_no)| token_to_string(token, *line_no))
            .collect();
        
        assert_eq!(output, vec![
            "INT int at Line 2.".to_string(),
            "IDENT func at Line 2.".to_string(),
            "L_PAREN ( at Line 2.".to_string(),
            "INT int at Line 2.".to_string(),
            "IDENT arg at Line 2.".to_string(),
            "R_PAREN ) at Line 2.".to_string(),
            "L_BRACE { at Line 2.".to_string(),
            "INT int at Line 3.".to_string(),
            "IDENT l at Line 3.".to_string(),
            "SEMICOLON ; at Line 3.".to_string(),
            "IDENT l at Line 4.".to_string(),
            "ASSIGN = at Line 4.".to_string(),
            "MINUS - at Line 4.".to_string(),
            "MINUS - at Line 4.".to_string(),
            "MINUS - at Line 4.".to_string(),
            "IDENT arg at Line 4.".to_string(),
            "SEMICOLON ; at Line 4.".to_string(),
            "RETURN return at Line 5.".to_string(),
            "IDENT l at Line 5.".to_string(),
            "SEMICOLON ; at Line 5.".to_string(),
            "R_BRACE } at Line 6.".to_string(),
            "INT int at Line 8.".to_string(),
            "IDENT main at Line 8.".to_string(),
            "L_PAREN ( at Line 8.".to_string(),
            "R_PAREN ) at Line 8.".to_string(),
            "L_BRACE { at Line 8.".to_string(),
            "INT int at Line 9.".to_string(),
            "IDENT x at Line 9.".to_string(),
            "COMMA , at Line 9.".to_string(),
            "IDENT y at Line 9.".to_string(),
            "SEMICOLON ; at Line 9.".to_string(),
            "IDENT x at Line 10.".to_string(),
            "ASSIGN = at Line 10.".to_string(),
            "INTEGER_CONST 2 at Line 10.".to_string(),
            "SEMICOLON ; at Line 10.".to_string(),
            "IDENT y at Line 11.".to_string(),
            "ASSIGN = at Line 11.".to_string(),
            "INTEGER_CONST 1 at Line 11.".to_string(),
            "SEMICOLON ; at Line 11.".to_string(),
            "IDENT x at Line 12.".to_string(),
            "ASSIGN = at Line 12.".to_string(),
            "IDENT x at Line 12.".to_string(),
            "MINUS - at Line 12.".to_string(),
            "INTEGER_CONST 1 at Line 12.".to_string(),
            "PLUS + at Line 12.".to_string(),
            "IDENT y at Line 12.".to_string(),
            "SEMICOLON ; at Line 12.".to_string(),
            "IF if at Line 13.".to_string(),
            "L_PAREN ( at Line 13.".to_string(),
            "PLUS + at Line 13.".to_string(),
            "MINUS - at Line 13.".to_string(),
            "NOT ! at Line 13.".to_string(),
            "NOT ! at Line 13.".to_string(),
            "NOT ! at Line 13.".to_string(),
            "IDENT x at Line 13.".to_string(),
            "R_PAREN ) at Line 13.".to_string(),
            "L_BRACE { at Line 13.".to_string(),
            "IDENT x at Line 14.".to_string(),
            "ASSIGN = at Line 14.".to_string(),
            "MINUS - at Line 14.".to_string(),
            "MINUS - at Line 14.".to_string(),
            "MINUS - at Line 14.".to_string(),
            "INTEGER_CONST 2 at Line 14.".to_string(),
            "SEMICOLON ; at Line 14.".to_string(),
            "R_BRACE } at Line 15.".to_string(),
            "ELSE else at Line 16.".to_string(),
            "L_BRACE { at Line 16.".to_string(),
            "IDENT x at Line 17.".to_string(),
            "ASSIGN = at Line 17.".to_string(),
            "INTEGER_CONST 1 at Line 17.".to_string(),
            "PLUS + at Line 17.".to_string(),
            "PLUS + at Line 17.".to_string(),
            "IDENT y at Line 17.".to_string(),
            "SEMICOLON ; at Line 17.".to_string(),
            "R_BRACE } at Line 18.".to_string(),
            "IDENT func at Line 19.".to_string(),
            "L_PAREN ( at Line 19.".to_string(),
            "IDENT x at Line 19.".to_string(),
            "R_PAREN ) at Line 19.".to_string(),
            "SEMICOLON ; at Line 19.".to_string(),
            "RETURN return at Line 20.".to_string(),
            "INTEGER_CONST 0 at Line 20.".to_string(),
            "SEMICOLON ; at Line 20.".to_string(),
            "R_BRACE } at Line 21.".to_string(),
        ]);
    }

    // 测试4: 样例四 - 数组和多进制常量
    #[test]
    fn test_sample_4() {
        let input = r#"
int array()
{
    int arr[10] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};

    int a1 = 0, a2 = 3, a3 = 5, a4 = 7, a5 = 9, a6 = 1, a7 = 2, a8 = 4,
        a9 = 6;

    return arr[a1] + arr[a2] + arr[a3] + arr[a4] + arr[a7] + arr[a8];
}

int main()
{
    int q = 1, r = 2, s = 04, t = 0x7, u = 0xA, v = 0xb, w = 0xcD, x = 077;

    int sum1 = q + r + s + t + u + v + w + x;

    int sum2 = array();

    int sum3 = sum1 + sum2;

    return 0;
}
"#;
        let result = tokenize(input);
        assert!(result.is_ok());
        
        let tokens = result.unwrap();
        let output: Vec<String> = tokens.iter()
            .map(|(token, line_no)| token_to_string(token, *line_no))
            .collect();
        
        assert_eq!(output, vec![
            "INT int at Line 2.".to_string(),
            "IDENT array at Line 2.".to_string(),
            "L_PAREN ( at Line 2.".to_string(),
            "R_PAREN ) at Line 2.".to_string(),
            "L_BRACE { at Line 3.".to_string(),
            "INT int at Line 4.".to_string(),
            "IDENT arr at Line 4.".to_string(),
            "L_BRACKT [ at Line 4.".to_string(),
            "INTEGER_CONST 10 at Line 4.".to_string(),
            "R_BRACKT ] at Line 4.".to_string(),
            "ASSIGN = at Line 4.".to_string(),
            "L_BRACE { at Line 4.".to_string(),
            "INTEGER_CONST 0 at Line 4.".to_string(),
            "COMMA , at Line 4.".to_string(),
            "INTEGER_CONST 1 at Line 4.".to_string(),
            "COMMA , at Line 4.".to_string(),
            "INTEGER_CONST 2 at Line 4.".to_string(),
            "COMMA , at Line 4.".to_string(),
            "INTEGER_CONST 3 at Line 4.".to_string(),
            "COMMA , at Line 4.".to_string(),
            "INTEGER_CONST 4 at Line 4.".to_string(),
            "COMMA , at Line 4.".to_string(),
            "INTEGER_CONST 5 at Line 4.".to_string(),
            "COMMA , at Line 4.".to_string(),
            "INTEGER_CONST 6 at Line 4.".to_string(),
            "COMMA , at Line 4.".to_string(),
            "INTEGER_CONST 7 at Line 4.".to_string(),
            "COMMA , at Line 4.".to_string(),
            "INTEGER_CONST 8 at Line 4.".to_string(),
            "COMMA , at Line 4.".to_string(),
            "INTEGER_CONST 9 at Line 4.".to_string(),
            "R_BRACE } at Line 4.".to_string(),
            "SEMICOLON ; at Line 4.".to_string(),
            "INT int at Line 6.".to_string(),
            "IDENT a1 at Line 6.".to_string(),
            "ASSIGN = at Line 6.".to_string(),
            "INTEGER_CONST 0 at Line 6.".to_string(),
            "COMMA , at Line 6.".to_string(),
            "IDENT a2 at Line 6.".to_string(),
            "ASSIGN = at Line 6.".to_string(),
            "INTEGER_CONST 3 at Line 6.".to_string(),
            "COMMA , at Line 6.".to_string(),
            "IDENT a3 at Line 6.".to_string(),
            "ASSIGN = at Line 6.".to_string(),
            "INTEGER_CONST 5 at Line 6.".to_string(),
            "COMMA , at Line 6.".to_string(),
            "IDENT a4 at Line 6.".to_string(),
            "ASSIGN = at Line 6.".to_string(),
            "INTEGER_CONST 7 at Line 6.".to_string(),
            "COMMA , at Line 6.".to_string(),
            "IDENT a5 at Line 6.".to_string(),
            "ASSIGN = at Line 6.".to_string(),
            "INTEGER_CONST 9 at Line 6.".to_string(),
            "COMMA , at Line 6.".to_string(),
            "IDENT a6 at Line 6.".to_string(),
            "ASSIGN = at Line 6.".to_string(),
            "INTEGER_CONST 1 at Line 6.".to_string(),
            "COMMA , at Line 6.".to_string(),
            "IDENT a7 at Line 6.".to_string(),
            "ASSIGN = at Line 6.".to_string(),
            "INTEGER_CONST 2 at Line 6.".to_string(),
            "COMMA , at Line 6.".to_string(),
            "IDENT a8 at Line 6.".to_string(),
            "ASSIGN = at Line 6.".to_string(),
            "INTEGER_CONST 4 at Line 6.".to_string(),
            "COMMA , at Line 6.".to_string(),
            "IDENT a9 at Line 7.".to_string(),
            "ASSIGN = at Line 7.".to_string(),
            "INTEGER_CONST 6 at Line 7.".to_string(),
            "SEMICOLON ; at Line 7.".to_string(),
            "RETURN return at Line 9.".to_string(),
            "IDENT arr at Line 9.".to_string(),
            "L_BRACKT [ at Line 9.".to_string(),
            "IDENT a1 at Line 9.".to_string(),
            "R_BRACKT ] at Line 9.".to_string(),
            "PLUS + at Line 9.".to_string(),
            "IDENT arr at Line 9.".to_string(),
            "L_BRACKT [ at Line 9.".to_string(),
            "IDENT a2 at Line 9.".to_string(),
            "R_BRACKT ] at Line 9.".to_string(),
            "PLUS + at Line 9.".to_string(),
            "IDENT arr at Line 9.".to_string(),
            "L_BRACKT [ at Line 9.".to_string(),
            "IDENT a3 at Line 9.".to_string(),
            "R_BRACKT ] at Line 9.".to_string(),
            "PLUS + at Line 9.".to_string(),
            "IDENT arr at Line 9.".to_string(),
            "L_BRACKT [ at Line 9.".to_string(),
            "IDENT a4 at Line 9.".to_string(),
            "R_BRACKT ] at Line 9.".to_string(),
            "PLUS + at Line 9.".to_string(),
            "IDENT arr at Line 9.".to_string(),
            "L_BRACKT [ at Line 9.".to_string(),
            "IDENT a7 at Line 9.".to_string(),
            "R_BRACKT ] at Line 9.".to_string(),
            "PLUS + at Line 9.".to_string(),
            "IDENT arr at Line 9.".to_string(),
            "L_BRACKT [ at Line 9.".to_string(),
            "IDENT a8 at Line 9.".to_string(),
            "R_BRACKT ] at Line 9.".to_string(),
            "SEMICOLON ; at Line 9.".to_string(),
            "R_BRACE } at Line 10.".to_string(),
            "INT int at Line 12.".to_string(),
            "IDENT main at Line 12.".to_string(),
            "L_PAREN ( at Line 12.".to_string(),
            "R_PAREN ) at Line 12.".to_string(),
            "L_BRACE { at Line 13.".to_string(),
            "INT int at Line 14.".to_string(),
            "IDENT q at Line 14.".to_string(),
            "ASSIGN = at Line 14.".to_string(),
            "INTEGER_CONST 1 at Line 14.".to_string(),
            "COMMA , at Line 14.".to_string(),
            "IDENT r at Line 14.".to_string(),
            "ASSIGN = at Line 14.".to_string(),
            "INTEGER_CONST 2 at Line 14.".to_string(),
            "COMMA , at Line 14.".to_string(),
            "IDENT s at Line 14.".to_string(),
            "ASSIGN = at Line 14.".to_string(),
            "INTEGER_CONST 4 at Line 14.".to_string(),
            "COMMA , at Line 14.".to_string(),
            "IDENT t at Line 14.".to_string(),
            "ASSIGN = at Line 14.".to_string(),
            "INTEGER_CONST 7 at Line 14.".to_string(),
            "COMMA , at Line 14.".to_string(),
            "IDENT u at Line 14.".to_string(),
            "ASSIGN = at Line 14.".to_string(),
            "INTEGER_CONST 10 at Line 14.".to_string(),
            "COMMA , at Line 14.".to_string(),
            "IDENT v at Line 14.".to_string(),
            "ASSIGN = at Line 14.".to_string(),
            "INTEGER_CONST 11 at Line 14.".to_string(),
            "COMMA , at Line 14.".to_string(),
            "IDENT w at Line 14.".to_string(),
            "ASSIGN = at Line 14.".to_string(),
            "INTEGER_CONST 205 at Line 14.".to_string(),
            "COMMA , at Line 14.".to_string(),
            "IDENT x at Line 14.".to_string(),
            "ASSIGN = at Line 14.".to_string(),
            "INTEGER_CONST 63 at Line 14.".to_string(),
            "SEMICOLON ; at Line 14.".to_string(),
            "INT int at Line 16.".to_string(),
            "IDENT sum1 at Line 16.".to_string(),
            "ASSIGN = at Line 16.".to_string(),
            "IDENT q at Line 16.".to_string(),
            "PLUS + at Line 16.".to_string(),
            "IDENT r at Line 16.".to_string(),
            "PLUS + at Line 16.".to_string(),
            "IDENT s at Line 16.".to_string(),
            "PLUS + at Line 16.".to_string(),
            "IDENT t at Line 16.".to_string(),
            "PLUS + at Line 16.".to_string(),
            "IDENT u at Line 16.".to_string(),
            "PLUS + at Line 16.".to_string(),
            "IDENT v at Line 16.".to_string(),
            "PLUS + at Line 16.".to_string(),
            "IDENT w at Line 16.".to_string(),
            "PLUS + at Line 16.".to_string(),
            "IDENT x at Line 16.".to_string(),
            "SEMICOLON ; at Line 16.".to_string(),
            "INT int at Line 18.".to_string(),
            "IDENT sum2 at Line 18.".to_string(),
            "ASSIGN = at Line 18.".to_string(),
            "IDENT array at Line 18.".to_string(),
            "L_PAREN ( at Line 18.".to_string(),
            "R_PAREN ) at Line 18.".to_string(),
            "SEMICOLON ; at Line 18.".to_string(),
            "INT int at Line 20.".to_string(),
            "IDENT sum3 at Line 20.".to_string(),
            "ASSIGN = at Line 20.".to_string(),
            "IDENT sum1 at Line 20.".to_string(),
            "PLUS + at Line 20.".to_string(),
            "IDENT sum2 at Line 20.".to_string(),
            "SEMICOLON ; at Line 20.".to_string(),
            "RETURN return at Line 22.".to_string(),
            "INTEGER_CONST 0 at Line 22.".to_string(),
            "SEMICOLON ; at Line 22.".to_string(),
            "R_BRACE } at Line 23.".to_string(),
        ]);
    }

    // 测试5: 关键字识别
    #[test]
    fn test_keywords() {
        let input = "const int void if else while break continue return";
        let result = tokenize(input);
        assert!(result.is_ok());
        
        let tokens = result.unwrap();
        let output: Vec<String> = tokens.iter()
            .map(|(token, line_no)| token_to_string(token, *line_no))
            .collect();
        
        assert_eq!(output, vec![
            "CONST const at Line 1.".to_string(),
            "INT int at Line 1.".to_string(),
            "VOID void at Line 1.".to_string(),
            "IF if at Line 1.".to_string(),
            "ELSE else at Line 1.".to_string(),
            "WHILE while at Line 1.".to_string(),
            "BREAK break at Line 1.".to_string(),
            "CONTINUE continue at Line 1.".to_string(),
            "RETURN return at Line 1.".to_string(),
        ]);
    }

    // 测试6: 运算符识别
    #[test]
    fn test_operators() {
        let input = "+ - * / % = == != < > <= >= ! && ||";
        let result = tokenize(input);
        assert!(result.is_ok());
        
        let tokens = result.unwrap();
        let output: Vec<String> = tokens.iter()
            .map(|(token, line_no)| token_to_string(token, *line_no))
            .collect();
        
        assert_eq!(output, vec![
            "PLUS + at Line 1.".to_string(),
            "MINUS - at Line 1.".to_string(),
            "MUL * at Line 1.".to_string(),
            "DIV / at Line 1.".to_string(),
            "MOD % at Line 1.".to_string(),
            "ASSIGN = at Line 1.".to_string(),
            "EQ == at Line 1.".to_string(),
            "NEQ != at Line 1.".to_string(),
            "LT < at Line 1.".to_string(),
            "GT > at Line 1.".to_string(),
            "LE <= at Line 1.".to_string(),
            "GE >= at Line 1.".to_string(),
            "NOT ! at Line 1.".to_string(),
            "AND && at Line 1.".to_string(),
            "OR || at Line 1.".to_string(),
        ]);
    }

    // 测试7: 标点符号识别
    #[test]
    fn test_punctuation() {
        let input = "( ) { } [ ] , ;";
        let result = tokenize(input);
        assert!(result.is_ok());
        
        let tokens = result.unwrap();
        let output: Vec<String> = tokens.iter()
            .map(|(token, line_no)| token_to_string(token, *line_no))
            .collect();
        
        assert_eq!(output, vec![
            "L_PAREN ( at Line 1.".to_string(),
            "R_PAREN ) at Line 1.".to_string(),
            "L_BRACE { at Line 1.".to_string(),
            "R_BRACE } at Line 1.".to_string(),
            "L_BRACKT [ at Line 1.".to_string(),
            "R_BRACKT ] at Line 1.".to_string(),
            "COMMA , at Line 1.".to_string(),
            "SEMICOLON ; at Line 1.".to_string(),
        ]);
    }

    // 测试8: 标识符识别 - 各种合法情况
    #[test]
    fn test_identifiers() {
        let input = "abc _abc abc123 _abc123 ABC a_b_c a1b2c3";
        let result = tokenize(input);
        assert!(result.is_ok());
        
        let tokens = result.unwrap();
        let output: Vec<String> = tokens.iter()
            .map(|(token, line_no)| token_to_string(token, *line_no))
            .collect();
        
        assert_eq!(output, vec![
            "IDENT abc at Line 1.".to_string(),
            "IDENT _abc at Line 1.".to_string(),
            "IDENT abc123 at Line 1.".to_string(),
            "IDENT _abc123 at Line 1.".to_string(),
            "IDENT ABC at Line 1.".to_string(),
            "IDENT a_b_c at Line 1.".to_string(),
            "IDENT a1b2c3 at Line 1.".to_string(),
        ]);
    }

    // 测试9: 十进制整数常量
    #[test]
    fn test_decimal_integers() {
        let input = "0 1 123 999999 000123";
        let result = tokenize(input);
        assert!(result.is_ok());
        
        let tokens = result.unwrap();
        let output: Vec<String> = tokens.iter()
            .map(|(token, line_no)| token_to_string(token, *line_no))
            .collect();
        
        assert_eq!(output, vec![
            "INTEGER_CONST 0 at Line 1.".to_string(),
            "INTEGER_CONST 1 at Line 1.".to_string(),
            "INTEGER_CONST 123 at Line 1.".to_string(),
            "INTEGER_CONST 999999 at Line 1.".to_string(),
            "INTEGER_CONST 123 at Line 1.".to_string(),
        ]);
    }

    // 测试10: 八进制整数常量
    #[test]
    fn test_octal_integers() {
        let input = "00 01 0123 0777";
        let result = tokenize(input);
        assert!(result.is_ok());
        
        let tokens = result.unwrap();
        let output: Vec<String> = tokens.iter()
            .map(|(token, line_no)| token_to_string(token, *line_no))
            .collect();
        
        assert_eq!(output, vec![
            "INTEGER_CONST 0 at Line 1.".to_string(),
            "INTEGER_CONST 1 at Line 1.".to_string(),
            "INTEGER_CONST 83 at Line 1.".to_string(),
            "INTEGER_CONST 511 at Line 1.".to_string(),
        ]);
    }

    // 测试11: 十六进制整数常量
    #[test]
    fn test_hexadecimal_integers() {
        let input = "0x0 0x1 0x123 0xABCD 0xabcd 0X123 0x000123";
        let result = tokenize(input);
        assert!(result.is_ok());
        
        let tokens = result.unwrap();
        let output: Vec<String> = tokens.iter()
            .map(|(token, line_no)| token_to_string(token, *line_no))
            .collect();
        
        assert_eq!(output, vec![
            "INTEGER_CONST 0 at Line 1.".to_string(),
            "INTEGER_CONST 1 at Line 1.".to_string(),
            "INTEGER_CONST 291 at Line 1.".to_string(),
            "INTEGER_CONST 43981 at Line 1.".to_string(),
            "INTEGER_CONST 43981 at Line 1.".to_string(),
            "INTEGER_CONST 291 at Line 1.".to_string(),
            "INTEGER_CONST 291 at Line 1.".to_string(),
        ]);
    }

    // 测试12: 空白符处理
    #[test]
    fn test_whitespace() {
        let input = "int\tx\n=\r\n123;";
        let result = tokenize(input);
        assert!(result.is_ok());
        
        let tokens = result.unwrap();
        let output: Vec<String> = tokens.iter()
            .map(|(token, line_no)| token_to_string(token, *line_no))
            .collect();
        
        assert_eq!(output, vec![
            "INT int at Line 1.".to_string(),
            "IDENT x at Line 1.".to_string(),
            "ASSIGN = at Line 2.".to_string(),
            "INTEGER_CONST 123 at Line 3.".to_string(),
            "SEMICOLON ; at Line 3.".to_string(),
        ]);
    }

    // 测试13: 单行注释处理
    #[test]
    fn test_line_comments() {
        let input = "int x = 1; // 这是注释\nint y = 2; // 另一个注释";
        let result = tokenize(input);
        assert!(result.is_ok());
        
        let tokens = result.unwrap();
        let output: Vec<String> = tokens.iter()
            .map(|(token, line_no)| token_to_string(token, *line_no))
            .collect();
        
        assert_eq!(output, vec![
            "INT int at Line 1.".to_string(),
            "IDENT x at Line 1.".to_string(),
            "ASSIGN = at Line 1.".to_string(),
            "INTEGER_CONST 1 at Line 1.".to_string(),
            "SEMICOLON ; at Line 1.".to_string(),
            "INT int at Line 2.".to_string(),
            "IDENT y at Line 2.".to_string(),
            "ASSIGN = at Line 2.".to_string(),
            "INTEGER_CONST 2 at Line 2.".to_string(),
            "SEMICOLON ; at Line 2.".to_string(),
        ]);
    }

    // 测试14: 多行注释处理
    #[test]
    fn test_multiline_comments() {
        let input = "int x = 1; /* 这是\n多行注释 */ int y = 2;";
        let result = tokenize(input);
        assert!(result.is_ok());
        
        let tokens = result.unwrap();
        let output: Vec<String> = tokens.iter()
            .map(|(token, line_no)| token_to_string(token, *line_no))
            .collect();
        
        assert_eq!(output, vec![
            "INT int at Line 1.".to_string(),
            "IDENT x at Line 1.".to_string(),
            "ASSIGN = at Line 1.".to_string(),
            "INTEGER_CONST 1 at Line 1.".to_string(),
            "SEMICOLON ; at Line 1.".to_string(),
            "INT int at Line 2.".to_string(),
            "IDENT y at Line 2.".to_string(),
            "ASSIGN = at Line 2.".to_string(),
            "INTEGER_CONST 2 at Line 2.".to_string(),
            "SEMICOLON ; at Line 2.".to_string(),
        ]);
    }

    // 测试15: 复杂表达式
    #[test]
    fn test_complex_expressions() {
        let input = "int result = (a + b) * c - d / e % f;";
        let result = tokenize(input);
        assert!(result.is_ok());
        
        let tokens = result.unwrap();
        let output: Vec<String> = tokens.iter()
            .map(|(token, line_no)| token_to_string(token, *line_no))
            .collect();
        
        assert_eq!(output, vec![
            "INT int at Line 1.".to_string(),
            "IDENT result at Line 1.".to_string(),
            "ASSIGN = at Line 1.".to_string(),
            "L_PAREN ( at Line 1.".to_string(),
            "IDENT a at Line 1.".to_string(),
            "PLUS + at Line 1.".to_string(),
            "IDENT b at Line 1.".to_string(),
            "R_PAREN ) at Line 1.".to_string(),
            "MUL * at Line 1.".to_string(),
            "IDENT c at Line 1.".to_string(),
            "MINUS - at Line 1.".to_string(),
            "IDENT d at Line 1.".to_string(),
            "DIV / at Line 1.".to_string(),
            "IDENT e at Line 1.".to_string(),
            "MOD % at Line 1.".to_string(),
            "IDENT f at Line 1.".to_string(),
            "SEMICOLON ; at Line 1.".to_string(),
        ]);
    }

    // 测试16: 条件语句
    #[test]
    fn test_conditional_statements() {
        let input = "if (x == y && z != w) { return 1; } else { return 0; }";
        let result = tokenize(input);
        assert!(result.is_ok());
        
        let tokens = result.unwrap();
        let output: Vec<String> = tokens.iter()
            .map(|(token, line_no)| token_to_string(token, *line_no))
            .collect();
        
        assert_eq!(output, vec![
            "IF if at Line 1.".to_string(),
            "L_PAREN ( at Line 1.".to_string(),
            "IDENT x at Line 1.".to_string(),
            "EQ == at Line 1.".to_string(),
            "IDENT y at Line 1.".to_string(),
            "AND && at Line 1.".to_string(),
            "IDENT z at Line 1.".to_string(),
            "NEQ != at Line 1.".to_string(),
            "IDENT w at Line 1.".to_string(),
            "R_PAREN ) at Line 1.".to_string(),
            "L_BRACE { at Line 1.".to_string(),
            "RETURN return at Line 1.".to_string(),
            "INTEGER_CONST 1 at Line 1.".to_string(),
            "SEMICOLON ; at Line 1.".to_string(),
            "R_BRACE } at Line 1.".to_string(),
            "ELSE else at Line 1.".to_string(),
            "L_BRACE { at Line 1.".to_string(),
            "RETURN return at Line 1.".to_string(),
            "INTEGER_CONST 0 at Line 1.".to_string(),
            "SEMICOLON ; at Line 1.".to_string(),
            "R_BRACE } at Line 1.".to_string(),
        ]);
    }

    // 测试17: 数组操作
    #[test]
    fn test_array_operations() {
        let input = "int arr[10] = {1, 2, 3}; arr[0] = 5;";
        let result = tokenize(input);
        assert!(result.is_ok());
        
        let tokens = result.unwrap();
        let output: Vec<String> = tokens.iter()
            .map(|(token, line_no)| token_to_string(token, *line_no))
            .collect();
        
        assert_eq!(output, vec![
            "INT int at Line 1.".to_string(),
            "IDENT arr at Line 1.".to_string(),
            "L_BRACKT [ at Line 1.".to_string(),
            "INTEGER_CONST 10 at Line 1.".to_string(),
            "R_BRACKT ] at Line 1.".to_string(),
            "ASSIGN = at Line 1.".to_string(),
            "L_BRACE { at Line 1.".to_string(),
            "INTEGER_CONST 1 at Line 1.".to_string(),
            "COMMA , at Line 1.".to_string(),
            "INTEGER_CONST 2 at Line 1.".to_string(),
            "COMMA , at Line 1.".to_string(),
            "INTEGER_CONST 3 at Line 1.".to_string(),
            "R_BRACE } at Line 1.".to_string(),
            "SEMICOLON ; at Line 1.".to_string(),
            "IDENT arr at Line 1.".to_string(),
            "L_BRACKT [ at Line 1.".to_string(),
            "INTEGER_CONST 0 at Line 1.".to_string(),
            "R_BRACKT ] at Line 1.".to_string(),
            "ASSIGN = at Line 1.".to_string(),
            "INTEGER_CONST 5 at Line 1.".to_string(),
            "SEMICOLON ; at Line 1.".to_string(),
        ]);
    }

    // 测试18: 数字后跟标识符
    #[test]
    fn test_number_followed_by_identifier() {
        let input = "int 2x = 08;";
        let result = tokenize(input);
        assert!(result.is_ok());
        
        let tokens = result.unwrap();
        let output: Vec<String> = tokens.iter()
            .map(|(token, line_no)| token_to_string(token, *line_no))
            .collect();
        
        assert_eq!(output, vec![
            "INT int at Line 1.".to_string(),
            "INTEGER_CONST 2 at Line 1.".to_string(),
            "IDENT x at Line 1.".to_string(),
            "ASSIGN = at Line 1.".to_string(),
            "INTEGER_CONST 0 at Line 1.".to_string(),
            "INTEGER_CONST 8 at Line 1.".to_string(),
            "SEMICOLON ; at Line 1.".to_string(),
        ]);
    }

    // 测试19: 边界情况 - 最大整数
    #[test]
    fn test_large_integers() {
        let input = "int x = 9223372036854775807; int y = 0x7FFFFFFFFFFFFFFF;";
        let result = tokenize(input);
        assert!(result.is_ok());
        
        let tokens = result.unwrap();
        let output: Vec<String> = tokens.iter()
            .map(|(token, line_no)| token_to_string(token, *line_no))
            .collect();
        
        assert_eq!(output, vec![
            "INT int at Line 1.".to_string(),
            "IDENT x at Line 1.".to_string(),
            "ASSIGN = at Line 1.".to_string(),
            "INTEGER_CONST 9223372036854775807 at Line 1.".to_string(),
            "SEMICOLON ; at Line 1.".to_string(),
            "INT int at Line 1.".to_string(),
            "IDENT y at Line 1.".to_string(),
            "ASSIGN = at Line 1.".to_string(),
            "INTEGER_CONST 9223372036854775807 at Line 1.".to_string(),
            "SEMICOLON ; at Line 1.".to_string(),
        ]);
    }

    // 测试20: 非法字符
    #[test]
    fn test_invalid_characters() {
        let input = "int x = @;";
        let result = tokenize(input);
        assert!(result.is_err());
        
        let (msg, line_no) = result.unwrap_err();
        assert_eq!(msg, "Mysterious character \"@\".");
        assert_eq!(line_no, 1);
    }

    // 测试21: 非法八进制/十六进制
    #[test]
    fn test_invalid_octal_hex() {
        let input = "int x = 08; int y = 0x;";
        let result = tokenize(input);
        assert!(result.is_ok());
        
        let tokens = result.unwrap();
        let output: Vec<String> = tokens.iter()
            .map(|(token, line_no)| token_to_string(token, *line_no))
            .collect();
        
        assert_eq!(output, vec![
            "INT int at Line 1.".to_string(),
            "IDENT x at Line 1.".to_string(),
            "ASSIGN = at Line 1.".to_string(),
            "INTEGER_CONST 0 at Line 1.".to_string(),
            "INTEGER_CONST 8 at Line 1.".to_string(),
            "SEMICOLON ; at Line 1.".to_string(),
            "INT int at Line 1.".to_string(),
            "IDENT y at Line 1.".to_string(),
            "ASSIGN = at Line 1.".to_string(),
            "INTEGER_CONST 0 at Line 1.".to_string(),
            "IDENT x at Line 1.".to_string(),
            "SEMICOLON ; at Line 1.".to_string(),
        ]);
    }

    // 测试22: 混合空白符和注释
    #[test]
    fn test_mixed_whitespace_and_comments() {
        let input = "int\tx\n=\r\n123; // 注释\n/* 多行\n注释 */\nint y = 456;";
        let result = tokenize(input);
        assert!(result.is_ok());
        
        let tokens = result.unwrap();
        let output: Vec<String> = tokens.iter()
            .map(|(token, line_no)| token_to_string(token, *line_no))
            .collect();
        
        assert_eq!(output, vec![
            "INT int at Line 1.".to_string(),
            "IDENT x at Line 1.".to_string(),
            "ASSIGN = at Line 2.".to_string(),
            "INTEGER_CONST 123 at Line 3.".to_string(),
            "SEMICOLON ; at Line 3.".to_string(),
            "INT int at Line 6.".to_string(),
            "IDENT y at Line 6.".to_string(),
            "ASSIGN = at Line 6.".to_string(),
            "INTEGER_CONST 456 at Line 6.".to_string(),
            "SEMICOLON ; at Line 6.".to_string(),
        ]);
    }

    // 测试23: 函数声明和调用
    #[test]
    fn test_function_declaration_and_call() {
        let input = "int func(int a, int b) { return a + b; } func(1, 2);";
        let result = tokenize(input);
        assert!(result.is_ok());
        
        let tokens = result.unwrap();
        let output: Vec<String> = tokens.iter()
            .map(|(token, line_no)| token_to_string(token, *line_no))
            .collect();
        
        assert_eq!(output, vec![
            "INT int at Line 1.".to_string(),
            "IDENT func at Line 1.".to_string(),
            "L_PAREN ( at Line 1.".to_string(),
            "INT int at Line 1.".to_string(),
            "IDENT a at Line 1.".to_string(),
            "COMMA , at Line 1.".to_string(),
            "INT int at Line 1.".to_string(),
            "IDENT b at Line 1.".to_string(),
            "R_PAREN ) at Line 1.".to_string(),
            "L_BRACE { at Line 1.".to_string(),
            "RETURN return at Line 1.".to_string(),
            "IDENT a at Line 1.".to_string(),
            "PLUS + at Line 1.".to_string(),
            "IDENT b at Line 1.".to_string(),
            "SEMICOLON ; at Line 1.".to_string(),
            "R_BRACE } at Line 1.".to_string(),
            "IDENT func at Line 1.".to_string(),
            "L_PAREN ( at Line 1.".to_string(),
            "INTEGER_CONST 1 at Line 1.".to_string(),
            "COMMA , at Line 1.".to_string(),
            "INTEGER_CONST 2 at Line 1.".to_string(),
            "R_PAREN ) at Line 1.".to_string(),
            "SEMICOLON ; at Line 1.".to_string(),
        ]);
    }

    // 测试24: 复杂嵌套结构
    #[test]
    fn test_complex_nested_structure() {
        let input = "if (x > 0) { if (y < 10) { arr[i] = x + y; } } else { return 0; }";
        let result = tokenize(input);
        assert!(result.is_ok());
        
        let tokens = result.unwrap();
        let output: Vec<String> = tokens.iter()
            .map(|(token, line_no)| token_to_string(token, *line_no))
            .collect();
        
        assert_eq!(output, vec![
            "IF if at Line 1.".to_string(),
            "L_PAREN ( at Line 1.".to_string(),
            "IDENT x at Line 1.".to_string(),
            "GT > at Line 1.".to_string(),
            "INTEGER_CONST 0 at Line 1.".to_string(),
            "R_PAREN ) at Line 1.".to_string(),
            "L_BRACE { at Line 1.".to_string(),
            "IF if at Line 1.".to_string(),
            "L_PAREN ( at Line 1.".to_string(),
            "IDENT y at Line 1.".to_string(),
            "LT < at Line 1.".to_string(),
            "INTEGER_CONST 10 at Line 1.".to_string(),
            "R_PAREN ) at Line 1.".to_string(),
            "L_BRACE { at Line 1.".to_string(),
            "IDENT arr at Line 1.".to_string(),
            "L_BRACKT [ at Line 1.".to_string(),
            "IDENT i at Line 1.".to_string(),
            "R_BRACKT ] at Line 1.".to_string(),
            "ASSIGN = at Line 1.".to_string(),
            "IDENT x at Line 1.".to_string(),
            "PLUS + at Line 1.".to_string(),
            "IDENT y at Line 1.".to_string(),
            "SEMICOLON ; at Line 1.".to_string(),
            "R_BRACE } at Line 1.".to_string(),
            "R_BRACE } at Line 1.".to_string(),
            "ELSE else at Line 1.".to_string(),
            "L_BRACE { at Line 1.".to_string(),
            "RETURN return at Line 1.".to_string(),
            "INTEGER_CONST 0 at Line 1.".to_string(),
            "SEMICOLON ; at Line 1.".to_string(),
            "R_BRACE } at Line 1.".to_string(),
        ]);
    }

    // 测试25: 关键字与标识符边界
    #[test]
    fn test_keyword_vs_identifier_boundaries() {
        let input = "ifelse intx void1 _if IF elseX while2 returnValue";
        let result = tokenize(input);
        assert!(result.is_ok());
        
        let tokens = result.unwrap();
        let output: Vec<String> = tokens.iter()
            .map(|(token, line_no)| token_to_string(token, *line_no))
            .collect();
        
        // 当前实现会将关键字和标识符分开识别
        assert_eq!(output, vec![
            "IF if at Line 1.".to_string(),
            "ELSE else at Line 1.".to_string(),
            "INT int at Line 1.".to_string(),
            "IDENT x at Line 1.".to_string(),
            "VOID void at Line 1.".to_string(),
            "INTEGER_CONST 1 at Line 1.".to_string(),
            "IDENT _if at Line 1.".to_string(),
            "IDENT IF at Line 1.".to_string(),
            "ELSE else at Line 1.".to_string(),
            "IDENT X at Line 1.".to_string(),
            "WHILE while at Line 1.".to_string(),
            "INTEGER_CONST 2 at Line 1.".to_string(),
            "RETURN return at Line 1.".to_string(),
            "IDENT Value at Line 1.".to_string(),
        ]);
    }

    // 测试26: 操作符"最长匹配"与紧凑无空格
    #[test]
    fn test_operator_longest_match_dense() {
        let input = "a<=b>=c==d!=e&&f||g<><=>=";
        let result = tokenize(input);
        assert!(result.is_ok());
        
        let tokens = result.unwrap();
        let output: Vec<String> = tokens.iter()
            .map(|(token, line_no)| token_to_string(token, *line_no))
            .collect();
        
        assert_eq!(output, vec![
            "IDENT a at Line 1.".to_string(),
            "LE <= at Line 1.".to_string(),
            "IDENT b at Line 1.".to_string(),
            "GE >= at Line 1.".to_string(),
            "IDENT c at Line 1.".to_string(),
            "EQ == at Line 1.".to_string(),
            "IDENT d at Line 1.".to_string(),
            "NEQ != at Line 1.".to_string(),
            "IDENT e at Line 1.".to_string(),
            "AND && at Line 1.".to_string(),
            "IDENT f at Line 1.".to_string(),
            "OR || at Line 1.".to_string(),
            "IDENT g at Line 1.".to_string(),
            "LT < at Line 1.".to_string(),
            "GT > at Line 1.".to_string(),
            "LE <= at Line 1.".to_string(),
            "GE >= at Line 1.".to_string(),
        ]);
    }

    // 测试27: 十六进制边界 - 无后续数字/非法十六进制字符
    #[test]
    fn test_hex_edge_cases() {
        let input = "int x = 0x; int y = 0X; int z = 0xG;";
        let result = tokenize(input);
        assert!(result.is_ok());
        
        let tokens = result.unwrap();
        let output: Vec<String> = tokens.iter()
            .map(|(token, line_no)| token_to_string(token, *line_no))
            .collect();
        
        assert_eq!(output, vec![
            "INT int at Line 1.".to_string(),
            "IDENT x at Line 1.".to_string(),
            "ASSIGN = at Line 1.".to_string(),
            "INTEGER_CONST 0 at Line 1.".to_string(),
            "IDENT x at Line 1.".to_string(),
            "SEMICOLON ; at Line 1.".to_string(),
            "INT int at Line 1.".to_string(),
            "IDENT y at Line 1.".to_string(),
            "ASSIGN = at Line 1.".to_string(),
            "INTEGER_CONST 0 at Line 1.".to_string(),
            "IDENT X at Line 1.".to_string(),
            "SEMICOLON ; at Line 1.".to_string(),
            "INT int at Line 1.".to_string(),
            "IDENT z at Line 1.".to_string(),
            "ASSIGN = at Line 1.".to_string(),
            "INTEGER_CONST 0 at Line 1.".to_string(),
            "IDENT xG at Line 1.".to_string(),
            "SEMICOLON ; at Line 1.".to_string(),
        ]);
    }

    // 测试28: 八进制边界 - 09/08 等拆分
    #[test]
    fn test_octal_split_cases() {
        let input = "int a=09; int b=08;";
        let result = tokenize(input);
        assert!(result.is_ok());
        
        let tokens = result.unwrap();
        let output: Vec<String> = tokens.iter()
            .map(|(token, line_no)| token_to_string(token, *line_no))
            .collect();
        
        assert_eq!(output, vec![
            "INT int at Line 1.".to_string(),
            "IDENT a at Line 1.".to_string(),
            "ASSIGN = at Line 1.".to_string(),
            "INTEGER_CONST 0 at Line 1.".to_string(),
            "INTEGER_CONST 9 at Line 1.".to_string(),
            "SEMICOLON ; at Line 1.".to_string(),
            "INT int at Line 1.".to_string(),
            "IDENT b at Line 1.".to_string(),
            "ASSIGN = at Line 1.".to_string(),
            "INTEGER_CONST 0 at Line 1.".to_string(),
            "INTEGER_CONST 8 at Line 1.".to_string(),
            "SEMICOLON ; at Line 1.".to_string(),
        ]);
    }

    // 测试29: 行注释结尾无换行
    #[test]
    fn test_line_comment_without_trailing_newline() {
        let input = "int x = 1; // comment without newline";
        let result = tokenize(input);
        assert!(result.is_ok());
        
        let tokens = result.unwrap();
        let output: Vec<String> = tokens.iter()
            .map(|(token, line_no)| token_to_string(token, *line_no))
            .collect();
        
        assert_eq!(output, vec![
            "INT int at Line 1.".to_string(),
            "IDENT x at Line 1.".to_string(),
            "ASSIGN = at Line 1.".to_string(),
            "INTEGER_CONST 1 at Line 1.".to_string(),
            "SEMICOLON ; at Line 1.".to_string(),
        ]);
    }

    // 测试30: 多行注释紧贴与跨行边界
    #[test]
    fn test_multiline_comment_adjacent_and_line_count() {
        let input = "int a/*comment*/=/*x*/1;/*\nblock\n*/int b=2;";
        let result = tokenize(input);
        assert!(result.is_ok());
        
        let tokens = result.unwrap();
        let output: Vec<String> = tokens.iter()
            .map(|(token, line_no)| token_to_string(token, *line_no))
            .collect();
        
        assert_eq!(output, vec![
            "INT int at Line 1.".to_string(),
            "IDENT a at Line 1.".to_string(),
            "ASSIGN = at Line 1.".to_string(),
            "INTEGER_CONST 1 at Line 1.".to_string(),
            "SEMICOLON ; at Line 1.".to_string(),
            "INT int at Line 3.".to_string(),
            "IDENT b at Line 3.".to_string(),
            "ASSIGN = at Line 3.".to_string(),
            "INTEGER_CONST 2 at Line 3.".to_string(),
            "SEMICOLON ; at Line 3.".to_string(),
        ]);
    }

    // 测试31: 紧凑的复杂表达式与括号/下标嵌套
    #[test]
    fn test_dense_complex_expression_with_indexes() {
        let input = "arr[0+foo[1]]={1,2,3};bar=(arr[2]+(x-y)%z);";
        let result = tokenize(input);
        assert!(result.is_ok());
        
        let tokens = result.unwrap();
        let output: Vec<String> = tokens.iter()
            .map(|(token, line_no)| token_to_string(token, *line_no))
            .collect();
        
        assert_eq!(output, vec![
            "IDENT arr at Line 1.".to_string(),
            "L_BRACKT [ at Line 1.".to_string(),
            "INTEGER_CONST 0 at Line 1.".to_string(),
            "PLUS + at Line 1.".to_string(),
            "IDENT foo at Line 1.".to_string(),
            "L_BRACKT [ at Line 1.".to_string(),
            "INTEGER_CONST 1 at Line 1.".to_string(),
            "R_BRACKT ] at Line 1.".to_string(),
            "R_BRACKT ] at Line 1.".to_string(),
            "ASSIGN = at Line 1.".to_string(),
            "L_BRACE { at Line 1.".to_string(),
            "INTEGER_CONST 1 at Line 1.".to_string(),
            "COMMA , at Line 1.".to_string(),
            "INTEGER_CONST 2 at Line 1.".to_string(),
            "COMMA , at Line 1.".to_string(),
            "INTEGER_CONST 3 at Line 1.".to_string(),
            "R_BRACE } at Line 1.".to_string(),
            "SEMICOLON ; at Line 1.".to_string(),
            "IDENT bar at Line 1.".to_string(),
            "ASSIGN = at Line 1.".to_string(),
            "L_PAREN ( at Line 1.".to_string(),
            "IDENT arr at Line 1.".to_string(),
            "L_BRACKT [ at Line 1.".to_string(),
            "INTEGER_CONST 2 at Line 1.".to_string(),
            "R_BRACKT ] at Line 1.".to_string(),
            "PLUS + at Line 1.".to_string(),
            "L_PAREN ( at Line 1.".to_string(),
            "IDENT x at Line 1.".to_string(),
            "MINUS - at Line 1.".to_string(),
            "IDENT y at Line 1.".to_string(),
            "R_PAREN ) at Line 1.".to_string(),
            "MOD % at Line 1.".to_string(),
            "IDENT z at Line 1.".to_string(),
            "R_PAREN ) at Line 1.".to_string(),
            "SEMICOLON ; at Line 1.".to_string(),
        ]);
    }

    // 测试32: 较长标识符与多下划线
    #[test]
    fn test_long_identifier_and_underscores() {
        let long_ident = "a________________________________________________________________";
        let input = format!("int {} = 123;", long_ident);
        let result = tokenize(&input);
        assert!(result.is_ok());
        
        let tokens = result.unwrap();
        let output: Vec<String> = tokens.iter()
            .map(|(token, line_no)| token_to_string(token, *line_no))
            .collect();
        
        assert_eq!(output, vec![
            "INT int at Line 1.".to_string(),
            format!("IDENT {} at Line 1.", long_ident),
            "ASSIGN = at Line 1.".to_string(),
            "INTEGER_CONST 123 at Line 1.".to_string(),
            "SEMICOLON ; at Line 1.".to_string(),
        ]);
    }

    // 测试33: 关键字与标识符相邻无空格
    #[test]
    fn test_keywords_adjacent_to_identifiers() {
        let input = "if(x)returny;intmain2(){return0;}";
        let result = tokenize(input);
        assert!(result.is_ok());
        
        let tokens = result.unwrap();
        let output: Vec<String> = tokens.iter()
            .map(|(token, line_no)| token_to_string(token, *line_no))
            .collect();
        
        assert_eq!(output, vec![
            "IF if at Line 1.".to_string(),
            "L_PAREN ( at Line 1.".to_string(),
            "IDENT x at Line 1.".to_string(),
            "R_PAREN ) at Line 1.".to_string(),
            "RETURN return at Line 1.".to_string(),
            "IDENT y at Line 1.".to_string(),
            "SEMICOLON ; at Line 1.".to_string(),
            "INT int at Line 1.".to_string(),
            "IDENT main2 at Line 1.".to_string(),
            "L_PAREN ( at Line 1.".to_string(),
            "R_PAREN ) at Line 1.".to_string(),
            "L_BRACE { at Line 1.".to_string(),
            "RETURN return at Line 1.".to_string(),
            "INTEGER_CONST 0 at Line 1.".to_string(),
            "SEMICOLON ; at Line 1.".to_string(),
            "R_BRACE } at Line 1.".to_string(),
        ]);
    }

    // 测试34: 注释内包含看似操作符/关键字的文本
    #[test]
    fn test_operators_and_keywords_inside_comments() {
        let input = "/* if && || != <= >= return */ int a=1; // else while \n a=a+1;";
        let result = tokenize(input);
        assert!(result.is_ok());
        
        let tokens = result.unwrap();
        let output: Vec<String> = tokens.iter()
            .map(|(token, line_no)| token_to_string(token, *line_no))
            .collect();
        
        assert_eq!(output, vec![
            "INT int at Line 1.".to_string(),
            "IDENT a at Line 1.".to_string(),
            "ASSIGN = at Line 1.".to_string(),
            "INTEGER_CONST 1 at Line 1.".to_string(),
            "SEMICOLON ; at Line 1.".to_string(),
            "IDENT a at Line 2.".to_string(),
            "ASSIGN = at Line 2.".to_string(),
            "IDENT a at Line 2.".to_string(),
            "PLUS + at Line 2.".to_string(),
            "INTEGER_CONST 1 at Line 2.".to_string(),
            "SEMICOLON ; at Line 2.".to_string(),
        ]);
    }

    // 测试35: 数字与标识符相邻的更多组合
    #[test]
    fn test_number_identifier_mixed_more_cases() {
        let input = "0x1abc 0x1abz 2then 0009z";
        let result = tokenize(input);
        assert!(result.is_ok());
        
        let tokens = result.unwrap();
        let output: Vec<String> = tokens.iter()
            .map(|(token, line_no)| token_to_string(token, *line_no))
            .collect();
        
        assert_eq!(output, vec![
            "INTEGER_CONST 6844 at Line 1.".to_string(), // 0x1abc
            "INTEGER_CONST 427 at Line 1.".to_string(),  // 0x1ab
            "IDENT z at Line 1.".to_string(),
            "INTEGER_CONST 2 at Line 1.".to_string(),
            "IDENT then at Line 1.".to_string(),
            "INTEGER_CONST 0 at Line 1.".to_string(),
            "INTEGER_CONST 9 at Line 1.".to_string(),
            "IDENT z at Line 1.".to_string(),
        ]);
    }

    // 测试36: 空输入
    #[test]
    fn test_empty_input() {
        let input = "";
        let result = tokenize(input);
        assert!(result.is_ok());
        
        let tokens = result.unwrap();
        assert!(tokens.is_empty());
    }

    // 测试37: 仅空白和注释
    #[test]
    fn test_only_whitespace_and_comments() {
        let input = "  \n\t\r\n// comment\n/* multi-line\ncomment */";
        let result = tokenize(input);
        assert!(result.is_ok());
        
        let tokens = result.unwrap();
        assert!(tokens.is_empty());
    }
    // 测试38: 多行复杂程序
    #[test]
    fn test_multi_line_complex_program() {
        let input = r#"
int factorial(int n) {
    if (n <= 1) {
        return 1;
    } else {
        return n * factorial(n - 1);
    }
}

int main() {
    int x = 5;
    int result = factorial(x);
    while (result > 0) {
        result = result - 1;
    }
    return 0;
}
"#;
        let result = tokenize(input);
        assert!(result.is_ok());
        
        let tokens = result.unwrap();
        let output: Vec<String> = tokens.iter()
            .map(|(token, line_no)| token_to_string(token, *line_no))
            .collect();
        
        // 验证函数声明和定义
        assert_eq!(output[0], "INT int at Line 2.");
        assert_eq!(output[1], "IDENT factorial at Line 2.");
        assert_eq!(output[2], "L_PAREN ( at Line 2.");
        assert_eq!(output[3], "INT int at Line 2.");
        assert_eq!(output[4], "IDENT n at Line 2.");
        assert_eq!(output[5], "R_PAREN ) at Line 2.");
        assert_eq!(output[6], "L_BRACE { at Line 2.");
        
        // 验证if语句
        assert_eq!(output[7], "IF if at Line 3.");
        assert_eq!(output[8], "L_PAREN ( at Line 3.");
        assert_eq!(output[9], "IDENT n at Line 3.");
        assert_eq!(output[10], "LE <= at Line 3.");
        assert_eq!(output[11], "INTEGER_CONST 1 at Line 3.");
        assert_eq!(output[12], "R_PAREN ) at Line 3.");
        assert_eq!(output[13], "L_BRACE { at Line 3.");
        assert_eq!(output[14], "RETURN return at Line 4.");
        assert_eq!(output[15], "INTEGER_CONST 1 at Line 4.");
        assert_eq!(output[16], "SEMICOLON ; at Line 4.");
        assert_eq!(output[17], "R_BRACE } at Line 5.");
        assert_eq!(output[18], "ELSE else at Line 5.");
        assert_eq!(output[19], "L_BRACE { at Line 5.");
        assert_eq!(output[20], "RETURN return at Line 6.");
        assert_eq!(output[21], "IDENT n at Line 6.");
        assert_eq!(output[22], "MUL * at Line 6.");
        assert_eq!(output[23], "IDENT factorial at Line 6.");
        assert_eq!(output[24], "L_PAREN ( at Line 6.");
        assert_eq!(output[25], "IDENT n at Line 6.");
        assert_eq!(output[26], "MINUS - at Line 6.");
        assert_eq!(output[27], "INTEGER_CONST 1 at Line 6.");
        assert_eq!(output[28], "R_PAREN ) at Line 6.");
        assert_eq!(output[29], "SEMICOLON ; at Line 6.");
        assert_eq!(output[30], "R_BRACE } at Line 7.");
        assert_eq!(output[31], "R_BRACE } at Line 8.");
        
        // 验证main函数
        assert_eq!(output[32], "INT int at Line 10.");
        assert_eq!(output[33], "IDENT main at Line 10.");
        assert_eq!(output[34], "L_PAREN ( at Line 10.");
        assert_eq!(output[35], "R_PAREN ) at Line 10.");
        assert_eq!(output[36], "L_BRACE { at Line 10.");
        
        // 验证变量声明和赋值
        assert_eq!(output[37], "INT int at Line 11.");
        assert_eq!(output[38], "IDENT x at Line 11.");
        assert_eq!(output[39], "ASSIGN = at Line 11.");
        assert_eq!(output[40], "INTEGER_CONST 5 at Line 11.");
        assert_eq!(output[41], "SEMICOLON ; at Line 11.");
        
        assert_eq!(output[42], "INT int at Line 12.");
        assert_eq!(output[43], "IDENT result at Line 12.");
        assert_eq!(output[44], "ASSIGN = at Line 12.");
        assert_eq!(output[45], "IDENT factorial at Line 12.");
        assert_eq!(output[46], "L_PAREN ( at Line 12.");
        assert_eq!(output[47], "IDENT x at Line 12.");
        assert_eq!(output[48], "R_PAREN ) at Line 12.");
        assert_eq!(output[49], "SEMICOLON ; at Line 12.");
        
        // 验证while循环
        assert_eq!(output[50], "WHILE while at Line 13.");
        assert_eq!(output[51], "L_PAREN ( at Line 13.");
        assert_eq!(output[52], "IDENT result at Line 13.");
        assert_eq!(output[53], "GT > at Line 13.");
        assert_eq!(output[54], "INTEGER_CONST 0 at Line 13.");
        assert_eq!(output[55], "R_PAREN ) at Line 13.");
        assert_eq!(output[56], "L_BRACE { at Line 13.");
        assert_eq!(output[57], "IDENT result at Line 14.");
        assert_eq!(output[58], "ASSIGN = at Line 14.");
        assert_eq!(output[59], "IDENT result at Line 14.");
        assert_eq!(output[60], "MINUS - at Line 14.");
        assert_eq!(output[61], "INTEGER_CONST 1 at Line 14.");
        assert_eq!(output[62], "SEMICOLON ; at Line 14.");
        assert_eq!(output[63], "R_BRACE } at Line 15.");
        assert_eq!(output[64], "RETURN return at Line 15.");
        assert_eq!(output[65], "INTEGER_CONST 0 at Line 15.");
        assert_eq!(output[66], "SEMICOLON ; at Line 15.");
        assert_eq!(output[67], "R_BRACE } at Line 17.");
    }

    // 测试39: 数组操作和指针解引用
    #[test]
    fn test_array_operations_and_dereference() {
        let input = "int arr[10]; arr[0] = 42; int *ptr = arr[5];";
        let result = tokenize(input);
        assert!(result.is_ok());
        
        let tokens = result.unwrap();
        let output: Vec<String> = tokens.iter()
            .map(|(token, line_no)| token_to_string(token, *line_no))
            .collect();
        
        // int arr[10];
        assert_eq!(output[0], "INT int at Line 1.");
        assert_eq!(output[1], "IDENT arr at Line 1.");
        assert_eq!(output[2], "L_BRACKT [ at Line 1.");
        assert_eq!(output[3], "INTEGER_CONST 10 at Line 1.");
        assert_eq!(output[4], "R_BRACKT ] at Line 1.");
        assert_eq!(output[5], "SEMICOLON ; at Line 1.");
        
        // arr[0] = 42;
        assert_eq!(output[6], "IDENT arr at Line 1.");
        assert_eq!(output[7], "L_BRACKT [ at Line 1.");
        assert_eq!(output[8], "INTEGER_CONST 0 at Line 1.");
        assert_eq!(output[9], "R_BRACKT ] at Line 1.");
        assert_eq!(output[10], "ASSIGN = at Line 1.");
        assert_eq!(output[11], "INTEGER_CONST 42 at Line 1.");
        assert_eq!(output[12], "SEMICOLON ; at Line 1.");
        
        // int *ptr = arr[5]; (注意：当前实现不支持&操作符)
        assert_eq!(output[13], "INT int at Line 1.");
        assert_eq!(output[14], "MUL * at Line 1.");
        assert_eq!(output[15], "IDENT ptr at Line 1.");
        assert_eq!(output[16], "ASSIGN = at Line 1.");
        assert_eq!(output[17], "IDENT arr at Line 1.");
        assert_eq!(output[18], "L_BRACKT [ at Line 1.");
        assert_eq!(output[19], "INTEGER_CONST 5 at Line 1.");
        assert_eq!(output[20], "R_BRACKT ] at Line 1.");
        assert_eq!(output[21], "SEMICOLON ; at Line 1.");
    }

    // 测试40: 复杂表达式和运算符优先级
    #[test]
    fn test_complex_expressions_and_operator_precedence() {
        let input = "int result = (a + b) * c - d / e % f;";
        let result = tokenize(input);
        assert!(result.is_ok());
        
        let tokens = result.unwrap();
        let output: Vec<String> = tokens.iter()
            .map(|(token, line_no)| token_to_string(token, *line_no))
            .collect();
        
        assert_eq!(output[0], "INT int at Line 1.");
        assert_eq!(output[1], "IDENT result at Line 1.");
        assert_eq!(output[2], "ASSIGN = at Line 1.");
        assert_eq!(output[3], "L_PAREN ( at Line 1.");
        assert_eq!(output[4], "IDENT a at Line 1.");
        assert_eq!(output[5], "PLUS + at Line 1.");
        assert_eq!(output[6], "IDENT b at Line 1.");
        assert_eq!(output[7], "R_PAREN ) at Line 1.");
        assert_eq!(output[8], "MUL * at Line 1.");
        assert_eq!(output[9], "IDENT c at Line 1.");
        assert_eq!(output[10], "MINUS - at Line 1.");
        assert_eq!(output[11], "IDENT d at Line 1.");
        assert_eq!(output[12], "DIV / at Line 1.");
        assert_eq!(output[13], "IDENT e at Line 1.");
        assert_eq!(output[14], "MOD % at Line 1.");
        assert_eq!(output[15], "IDENT f at Line 1.");
        assert_eq!(output[16], "SEMICOLON ; at Line 1.");
    }

    // 测试41: 逻辑运算符和比较运算符组合
    #[test]
    fn test_logical_and_comparison_operators() {
        let input = "if (x > 0 && y < 10 || z == 0) { flag = true; }";
        let result = tokenize(input);
        assert!(result.is_ok());
        
        let tokens = result.unwrap();
        let output: Vec<String> = tokens.iter()
            .map(|(token, line_no)| token_to_string(token, *line_no))
            .collect();
        
        assert_eq!(output[0], "IF if at Line 1.");
        assert_eq!(output[1], "L_PAREN ( at Line 1.");
        assert_eq!(output[2], "IDENT x at Line 1.");
        assert_eq!(output[3], "GT > at Line 1.");
        assert_eq!(output[4], "INTEGER_CONST 0 at Line 1.");
        assert_eq!(output[5], "AND && at Line 1.");
        assert_eq!(output[6], "IDENT y at Line 1.");
        assert_eq!(output[7], "LT < at Line 1.");
        assert_eq!(output[8], "INTEGER_CONST 10 at Line 1.");
        assert_eq!(output[9], "OR || at Line 1.");
        assert_eq!(output[10], "IDENT z at Line 1.");
        assert_eq!(output[11], "EQ == at Line 1.");
        assert_eq!(output[12], "INTEGER_CONST 0 at Line 1.");
        assert_eq!(output[13], "R_PAREN ) at Line 1.");
        assert_eq!(output[14], "L_BRACE { at Line 1.");
        assert_eq!(output[15], "IDENT flag at Line 1.");
        assert_eq!(output[16], "ASSIGN = at Line 1.");
        assert_eq!(output[17], "IDENT true at Line 1.");
        assert_eq!(output[18], "SEMICOLON ; at Line 1.");
        assert_eq!(output[19], "R_BRACE } at Line 1.");
    }

    // 测试42: 嵌套注释和复杂注释
    #[test]
    fn test_nested_and_complex_comments() {
        let input = r#"
// 这是单行注释
int x = 1; /* 这是多行注释
   可以跨越多行
   包含关键字: if, while, return
   包含操作符: +, -, *, /
*/
int y = 2; // 另一个单行注释
"#;
        let result = tokenize(input);
        assert!(result.is_ok());
        
        let tokens = result.unwrap();
        let output: Vec<String> = tokens.iter()
            .map(|(token, line_no)| token_to_string(token, *line_no))
            .collect();
        
        // 注释被忽略，只保留实际代码
        assert_eq!(output[0], "INT int at Line 3.");
        assert_eq!(output[1], "IDENT x at Line 3.");
        assert_eq!(output[2], "ASSIGN = at Line 3.");
        assert_eq!(output[3], "INTEGER_CONST 1 at Line 3.");
        assert_eq!(output[4], "SEMICOLON ; at Line 3.");
        assert_eq!(output[5], "INT int at Line 8.");
        assert_eq!(output[6], "IDENT y at Line 8.");
        assert_eq!(output[7], "ASSIGN = at Line 8.");
        assert_eq!(output[8], "INTEGER_CONST 2 at Line 8.");
        assert_eq!(output[9], "SEMICOLON ; at Line 8.");
    }

    // 测试43: 边界情况：最大整数和特殊字符
    #[test]
    fn test_edge_cases_max_integers_and_special_chars() {
        let input = "int max_int = 9223372036854775807; int min_int = 9223372036854775806;";
        let result = tokenize(input);
        assert!(result.is_ok());
        
        let tokens = result.unwrap();
        let output: Vec<String> = tokens.iter()
            .map(|(token, line_no)| token_to_string(token, *line_no))
            .collect();
        
        // 注意：当前实现不支持负数，所以改为正数
        assert_eq!(output[0], "INT int at Line 1.");
        assert_eq!(output[1], "IDENT max_int at Line 1.");
        assert_eq!(output[2], "ASSIGN = at Line 1.");
        assert_eq!(output[3], "INTEGER_CONST 9223372036854775807 at Line 1.");
        assert_eq!(output[4], "SEMICOLON ; at Line 1.");
        assert_eq!(output[5], "INT int at Line 1.");
        assert_eq!(output[6], "IDENT min_int at Line 1.");
        assert_eq!(output[7], "ASSIGN = at Line 1.");
        assert_eq!(output[8], "INTEGER_CONST 9223372036854775806 at Line 1.");
        assert_eq!(output[9], "SEMICOLON ; at Line 1.");
    }

    // 测试44: 标识符的各种组合和边界
    #[test]
    fn test_identifier_various_combinations_and_boundaries() {
        let input = "_ _a _1 _a1 a_ a1_ a_1 _a_b _1_2 a_b_1";
        let result = tokenize(input);
        assert!(result.is_ok());
        
        let tokens = result.unwrap();
        let output: Vec<String> = tokens.iter()
            .map(|(token, line_no)| token_to_string(token, *line_no))
            .collect();
        
        assert_eq!(output[0], "IDENT _ at Line 1.");
        assert_eq!(output[1], "IDENT _a at Line 1.");
        assert_eq!(output[2], "IDENT _1 at Line 1.");
        assert_eq!(output[3], "IDENT _a1 at Line 1.");
        assert_eq!(output[4], "IDENT a_ at Line 1.");
        assert_eq!(output[5], "IDENT a1_ at Line 1.");
        assert_eq!(output[6], "IDENT a_1 at Line 1.");
        assert_eq!(output[7], "IDENT _a_b at Line 1.");
        assert_eq!(output[8], "IDENT _1_2 at Line 1.");
        assert_eq!(output[9], "IDENT a_b_1 at Line 1.");
    }

    // 测试45: 操作符的边界情况和组合
    #[test]
    fn test_operator_edge_cases_and_combinations() {
        let input = "a+++b a---b a***b a===b a!!!b";
        let result = tokenize(input);
        assert!(result.is_ok());
        
        let tokens = result.unwrap();
        let output: Vec<String> = tokens.iter()
            .map(|(token, line_no)| token_to_string(token, *line_no))
            .collect();
        
        // a+++b -> a + + + b
        assert_eq!(output[0], "IDENT a at Line 1.");
        assert_eq!(output[1], "PLUS + at Line 1.");
        assert_eq!(output[2], "PLUS + at Line 1.");
        assert_eq!(output[3], "PLUS + at Line 1.");
        assert_eq!(output[4], "IDENT b at Line 1.");
        
        // a---b -> a - - - b
        assert_eq!(output[5], "IDENT a at Line 1.");
        assert_eq!(output[6], "MINUS - at Line 1.");
        assert_eq!(output[7], "MINUS - at Line 1.");
        assert_eq!(output[8], "MINUS - at Line 1.");
        assert_eq!(output[9], "IDENT b at Line 1.");
        
        // a***b -> a * * * b
        assert_eq!(output[10], "IDENT a at Line 1.");
        assert_eq!(output[11], "MUL * at Line 1.");
        assert_eq!(output[12], "MUL * at Line 1.");
        assert_eq!(output[13], "MUL * at Line 1.");
        assert_eq!(output[14], "IDENT b at Line 1.");
        
        // a===b -> a == = b (注意：===被解析为==和=)
        assert_eq!(output[15], "IDENT a at Line 1.");
        assert_eq!(output[16], "EQ == at Line 1.");
        assert_eq!(output[17], "ASSIGN = at Line 1.");
        assert_eq!(output[18], "IDENT b at Line 1.");
        
        // a!!!b -> a ! ! ! b
        assert_eq!(output[19], "IDENT a at Line 1.");
        assert_eq!(output[20], "NOT ! at Line 1.");
        assert_eq!(output[21], "NOT ! at Line 1.");
        assert_eq!(output[22], "NOT ! at Line 1.");
        assert_eq!(output[23], "IDENT b at Line 1.");
    }

    // 测试46: 错误处理和非法字符
    #[test]
    fn test_error_handling_and_invalid_characters() {
        let input = "int x = 5; int y = @; int z = #;";
        let result = tokenize(input);
        assert!(result.is_err());
        
        let (error_msg, line_no) = result.unwrap_err();
        assert!(error_msg.contains("Mysterious character"));
        assert_eq!(line_no, 1);
    }

    // 测试47: 性能测试：大量标识符和数字
    #[test]
    fn test_performance_large_number_of_tokens() {
        let mut input = String::new();
        for i in 0..1000 {
            input.push_str(&format!("int var{} = {}; ", i, i));
        }
        
        let result = tokenize(&input);
        assert!(result.is_ok());
        
        let tokens = result.unwrap();
        assert_eq!(tokens.len(), 5000); // 每个声明：int, varN, =, N, ;
        
        // 验证第一个和最后一个token
        let first_token = &tokens[0];
        let last_token = &tokens[tokens.len() - 1];
        
        assert_eq!(first_token.0, Token::Int);
        assert_eq!(last_token.0, Token::Semicolon);
    }

    // 测试48: 国际化字符和Unicode标识符
    #[test]
    fn test_international_characters_and_unicode_identifiers() {
        let input = "int 变量 = 42; int 函数名 = 123; int _中文_ = 456;";
        let result = tokenize(input);
        // 当前实现不支持Unicode标识符，所以会报错
        assert!(result.is_err());
        
        let (error_msg, line_no) = result.unwrap_err();
        assert!(error_msg.contains("Mysterious character"));
        assert_eq!(line_no, 1);
    }

    // 测试49: 边界情况：空字符串和特殊空白字符
    #[test]
    fn test_edge_cases_empty_strings_and_special_whitespace() {
        let input = "\"\" '' `` \t\n\r";
        let result = tokenize(input);
        // 当前实现不支持字符串字面量，所以会报错
        assert!(result.is_err());
        
        let (error_msg, line_no) = result.unwrap_err();
        assert!(error_msg.contains("Mysterious character"));
        assert_eq!(line_no, 1);
    }

    // 测试50: 综合测试：完整的编译器前端测试
    #[test]
    fn test_comprehensive_compiler_frontend() {
        let input = r#"
// 这是一个完整的SysY程序示例
const int MAX_SIZE = 100;
const int ARRAY_SIZE = 10;

int fibonacci(int n) {
    if (n <= 1) {
        return n;
    }
    return fibonacci(n - 1) + fibonacci(n - 2);
}

int main() {
    int numbers[ARRAY_SIZE];
    int i = 0;
    
    // 初始化数组
    while (i < ARRAY_SIZE) {
        numbers[i] = fibonacci(i);
        i = i + 1;
    }
    
    // 计算总和
    int sum = 0;
    i = 0;
    while (i < ARRAY_SIZE) {
        sum = sum + numbers[i];
        i = i + 1;
    }
    
    return sum;
}
"#;
        let result = tokenize(input);
        assert!(result.is_ok());
        
        let tokens = result.unwrap();
        let output: Vec<String> = tokens.iter()
            .map(|(token, line_no)| token_to_string(token, *line_no))
            .collect();
        
        // 验证常量声明
        assert_eq!(output[0], "CONST const at Line 3.");
        assert_eq!(output[1], "INT int at Line 3.");
        assert_eq!(output[2], "IDENT MAX_SIZE at Line 3.");
        assert_eq!(output[3], "ASSIGN = at Line 3.");
        assert_eq!(output[4], "INTEGER_CONST 100 at Line 3.");
        assert_eq!(output[5], "SEMICOLON ; at Line 3.");
        
        assert_eq!(output[6], "CONST const at Line 4.");
        assert_eq!(output[7], "INT int at Line 4.");
        assert_eq!(output[8], "IDENT ARRAY_SIZE at Line 4.");
        assert_eq!(output[9], "ASSIGN = at Line 4.");
        assert_eq!(output[10], "INTEGER_CONST 10 at Line 4.");
        assert_eq!(output[11], "SEMICOLON ; at Line 4.");
        
        // 验证函数定义
        assert_eq!(output[12], "INT int at Line 6.");
        assert_eq!(output[13], "IDENT fibonacci at Line 6.");
        assert_eq!(output[14], "L_PAREN ( at Line 6.");
        assert_eq!(output[15], "INT int at Line 6.");
        assert_eq!(output[16], "IDENT n at Line 6.");
        assert_eq!(output[17], "R_PAREN ) at Line 6.");
        assert_eq!(output[18], "L_BRACE { at Line 6.");
        
        // 验证函数体
        assert_eq!(output[19], "IF if at Line 7.");
        assert_eq!(output[20], "L_PAREN ( at Line 7.");
        assert_eq!(output[21], "IDENT n at Line 7.");
        assert_eq!(output[22], "LE <= at Line 7.");
        assert_eq!(output[23], "INTEGER_CONST 1 at Line 7.");
        assert_eq!(output[24], "R_PAREN ) at Line 7.");
        assert_eq!(output[25], "L_BRACE { at Line 7.");
        assert_eq!(output[26], "RETURN return at Line 8.");
        assert_eq!(output[27], "IDENT n at Line 8.");
        assert_eq!(output[28], "SEMICOLON ; at Line 8.");
        assert_eq!(output[29], "R_BRACE } at Line 9.");
        
        assert_eq!(output[30], "RETURN return at Line 10.");
        assert_eq!(output[31], "IDENT fibonacci at Line 10.");
        assert_eq!(output[32], "L_PAREN ( at Line 10.");
        assert_eq!(output[33], "IDENT n at Line 10.");
        assert_eq!(output[34], "MINUS - at Line 10.");
        assert_eq!(output[35], "INTEGER_CONST 1 at Line 10.");
        assert_eq!(output[36], "R_PAREN ) at Line 10.");
        assert_eq!(output[37], "PLUS + at Line 10.");
        assert_eq!(output[38], "IDENT fibonacci at Line 10.");
        assert_eq!(output[39], "L_PAREN ( at Line 10.");
        assert_eq!(output[40], "IDENT n at Line 10.");
        assert_eq!(output[41], "MINUS - at Line 10.");
        assert_eq!(output[42], "INTEGER_CONST 2 at Line 10.");
        assert_eq!(output[43], "R_PAREN ) at Line 10.");
        assert_eq!(output[44], "SEMICOLON ; at Line 10.");
        assert_eq!(output[45], "R_BRACE } at Line 11.");
        
        // 验证main函数
        assert_eq!(output[46], "INT int at Line 13.");
        assert_eq!(output[47], "IDENT main at Line 13.");
        assert_eq!(output[48], "L_PAREN ( at Line 13.");
        assert_eq!(output[49], "R_PAREN ) at Line 13.");
        assert_eq!(output[50], "L_BRACE { at Line 13.");
        
        // 验证数组声明和初始化
        assert_eq!(output[51], "INT int at Line 14.");
        assert_eq!(output[52], "IDENT numbers at Line 14.");
        assert_eq!(output[53], "L_BRACKT [ at Line 14.");
        assert_eq!(output[54], "IDENT ARRAY_SIZE at Line 14.");
        assert_eq!(output[55], "R_BRACKT ] at Line 14.");
        assert_eq!(output[56], "SEMICOLON ; at Line 14.");
        
        assert_eq!(output[57], "INT int at Line 15.");
        assert_eq!(output[58], "IDENT i at Line 15.");
        assert_eq!(output[59], "ASSIGN = at Line 15.");
        assert_eq!(output[60], "INTEGER_CONST 0 at Line 15.");
        assert_eq!(output[61], "SEMICOLON ; at Line 15.");
        
        // 验证while循环和数组操作
        assert_eq!(output[62], "WHILE while at Line 18.");
        assert_eq!(output[63], "L_PAREN ( at Line 18.");
        assert_eq!(output[64], "IDENT i at Line 18.");
        assert_eq!(output[65], "LT < at Line 18.");
        assert_eq!(output[66], "IDENT ARRAY_SIZE at Line 18.");
        assert_eq!(output[67], "R_PAREN ) at Line 18.");
        assert_eq!(output[68], "L_BRACE { at Line 18.");
        
        assert_eq!(output[69], "IDENT numbers at Line 19.");
        assert_eq!(output[70], "L_BRACKT [ at Line 19.");
        assert_eq!(output[71], "IDENT i at Line 19.");
        assert_eq!(output[72], "R_BRACKT ] at Line 19.");
        assert_eq!(output[73], "ASSIGN = at Line 19.");
        assert_eq!(output[74], "IDENT fibonacci at Line 19.");
        assert_eq!(output[75], "L_PAREN ( at Line 19.");
        assert_eq!(output[76], "IDENT i at Line 19.");
        assert_eq!(output[77], "R_PAREN ) at Line 19.");
        assert_eq!(output[78], "SEMICOLON ; at Line 19.");
        
        assert_eq!(output[79], "IDENT i at Line 20.");
        assert_eq!(output[80], "ASSIGN = at Line 20.");
        assert_eq!(output[81], "IDENT i at Line 20.");
        assert_eq!(output[82], "PLUS + at Line 20.");
        assert_eq!(output[83], "INTEGER_CONST 1 at Line 20.");
        assert_eq!(output[84], "SEMICOLON ; at Line 20.");
        assert_eq!(output[85], "R_BRACE } at Line 21.");
        
        // 验证计算总和部分
        assert_eq!(output[86], "INT int at Line 24.");
        assert_eq!(output[87], "IDENT sum at Line 24.");
        assert_eq!(output[88], "ASSIGN = at Line 24.");
        assert_eq!(output[89], "INTEGER_CONST 0 at Line 24.");
        assert_eq!(output[90], "SEMICOLON ; at Line 24.");
        
        assert_eq!(output[91], "IDENT i at Line 25.");
        assert_eq!(output[92], "ASSIGN = at Line 25.");
        assert_eq!(output[93], "INTEGER_CONST 0 at Line 25.");
        assert_eq!(output[94], "SEMICOLON ; at Line 25.");
        
        // 验证第二个while循环
        assert_eq!(output[95], "WHILE while at Line 26.");
        assert_eq!(output[96], "L_PAREN ( at Line 26.");
        assert_eq!(output[97], "IDENT i at Line 26.");
        assert_eq!(output[98], "LT < at Line 26.");
        assert_eq!(output[99], "IDENT ARRAY_SIZE at Line 26.");
        assert_eq!(output[100], "R_PAREN ) at Line 26.");
        assert_eq!(output[101], "L_BRACE { at Line 26.");
        
        assert_eq!(output[102], "IDENT sum at Line 27.");
        assert_eq!(output[103], "ASSIGN = at Line 27.");
        assert_eq!(output[104], "IDENT sum at Line 27.");
        assert_eq!(output[105], "PLUS + at Line 27.");
        assert_eq!(output[106], "IDENT numbers at Line 27.");
        assert_eq!(output[107], "L_BRACKT [ at Line 27.");
        assert_eq!(output[108], "IDENT i at Line 27.");
        assert_eq!(output[109], "R_BRACKT ] at Line 27.");
        assert_eq!(output[110], "SEMICOLON ; at Line 27.");
        
        assert_eq!(output[111], "IDENT i at Line 28.");
        assert_eq!(output[112], "ASSIGN = at Line 28.");
        assert_eq!(output[113], "IDENT i at Line 28.");
        assert_eq!(output[114], "PLUS + at Line 28.");
        assert_eq!(output[115], "INTEGER_CONST 1 at Line 28.");
        assert_eq!(output[116], "SEMICOLON ; at Line 28.");
        assert_eq!(output[117], "R_BRACE } at Line 29.");
        
        // 验证return语句
        assert_eq!(output[118], "RETURN return at Line 31.");
        assert_eq!(output[119], "IDENT sum at Line 31.");
        assert_eq!(output[120], "SEMICOLON ; at Line 31.");
        assert_eq!(output[121], "R_BRACE } at Line 32.");
    }

    // 测试51: 未闭合多行注释（应报错，行号为注释起始行）
    #[test]
    fn test_unclosed_multiline_comment() {
        let input = "int a = 1; /* unclosed\nint b = 2;";
        let result = tokenize(input);
        assert!(result.is_err());
        let (_msg, line_no) = result.unwrap_err();
        assert_eq!(line_no, 1);
    }

    // 测试52: 多重前导零的十进制解析（应输出十进制值）
    #[test]
    fn test_many_leading_zeros_decimalized() {
        let input = "int x=0000000000007;";
        let result = tokenize(input);
        assert!(result.is_ok());
        let tokens = result.unwrap();
        let output: Vec<String> = tokens.iter().map(|(t, l)| token_to_string(t, *l)).collect();
        assert_eq!(output, vec![
            "INT int at Line 1.".to_string(),
            "IDENT x at Line 1.".to_string(),
            "ASSIGN = at Line 1.".to_string(),
            "INTEGER_CONST 7 at Line 1.".to_string(),
            "SEMICOLON ; at Line 1.".to_string(),
        ]);
    }

    // 测试53: 超范围十六进制常量（解析应失败并返回错误）
    #[test]
    fn test_hex_overflow_error() {
        let input = "int x = 0xFFFFFFFFFFFFFFFFF;"; // 17个F，超出i64
        let result = tokenize(input);
        assert!(result.is_err());
        let (msg, line_no) = result.unwrap_err();
        assert!(msg.contains("Invalid hex number"));
        assert_eq!(line_no, 1);
    }

    // 测试54: 紧凑的函数与数组、括号深度嵌套组合
    #[test]
    fn test_dense_function_and_array_nesting() {
        let input = "f(g[0]+h(i[j]+k),arr[1+u*v])[x+y];";
        let result = tokenize(input);
        assert!(result.is_ok());
        let tokens = result.unwrap();
        let output: Vec<String> = tokens.iter().map(|(t, l)| token_to_string(t, *l)).collect();
        assert_eq!(output, vec![
            "IDENT f at Line 1.".to_string(),
            "L_PAREN ( at Line 1.".to_string(),
            "IDENT g at Line 1.".to_string(),
            "L_BRACKT [ at Line 1.".to_string(),
            "INTEGER_CONST 0 at Line 1.".to_string(),
            "R_BRACKT ] at Line 1.".to_string(),
            "PLUS + at Line 1.".to_string(),
            "IDENT h at Line 1.".to_string(),
            "L_PAREN ( at Line 1.".to_string(),
            "IDENT i at Line 1.".to_string(),
            "L_BRACKT [ at Line 1.".to_string(),
            "IDENT j at Line 1.".to_string(),
            "R_BRACKT ] at Line 1.".to_string(),
            "PLUS + at Line 1.".to_string(),
            "IDENT k at Line 1.".to_string(),
            "R_PAREN ) at Line 1.".to_string(),
            "COMMA , at Line 1.".to_string(),
            "IDENT arr at Line 1.".to_string(),
            "L_BRACKT [ at Line 1.".to_string(),
            "INTEGER_CONST 1 at Line 1.".to_string(),
            "PLUS + at Line 1.".to_string(),
            "IDENT u at Line 1.".to_string(),
            "MUL * at Line 1.".to_string(),
            "IDENT v at Line 1.".to_string(),
            "R_BRACKT ] at Line 1.".to_string(),
            "R_PAREN ) at Line 1.".to_string(),
            "L_BRACKT [ at Line 1.".to_string(),
            "IDENT x at Line 1.".to_string(),
            "PLUS + at Line 1.".to_string(),
            "IDENT y at Line 1.".to_string(),
            "R_BRACKT ] at Line 1.".to_string(),
            "SEMICOLON ; at Line 1.".to_string(),
        ]);
    }

    // 测试55: true/false 作为标识符使用
    #[test]
    fn test_true_false_as_identifiers() {
        let input = "int true=1; int false=0; if(true){false=false+1;}";
        let result = tokenize(input);
        assert!(result.is_ok());
        let tokens = result.unwrap();
        let output: Vec<String> = tokens.iter().map(|(t, l)| token_to_string(t, *l)).collect();
        assert_eq!(output, vec![
            "INT int at Line 1.".to_string(),
            "IDENT true at Line 1.".to_string(),
            "ASSIGN = at Line 1.".to_string(),
            "INTEGER_CONST 1 at Line 1.".to_string(),
            "SEMICOLON ; at Line 1.".to_string(),
            "INT int at Line 1.".to_string(),
            "IDENT false at Line 1.".to_string(),
            "ASSIGN = at Line 1.".to_string(),
            "INTEGER_CONST 0 at Line 1.".to_string(),
            "SEMICOLON ; at Line 1.".to_string(),
            "IF if at Line 1.".to_string(),
            "L_PAREN ( at Line 1.".to_string(),
            "IDENT true at Line 1.".to_string(),
            "R_PAREN ) at Line 1.".to_string(),
            "L_BRACE { at Line 1.".to_string(),
            "IDENT false at Line 1.".to_string(),
            "ASSIGN = at Line 1.".to_string(),
            "IDENT false at Line 1.".to_string(),
            "PLUS + at Line 1.".to_string(),
            "INTEGER_CONST 1 at Line 1.".to_string(),
            "SEMICOLON ; at Line 1.".to_string(),
            "R_BRACE } at Line 1.".to_string(),
        ]);
    }

    // 测试56: 仅注释且文件末尾无换行
    #[test]
    fn test_only_comments_without_trailing_newline() {
        let input = "// comment\n/* block */"; // 末尾无换行
        let result = tokenize(input);
        assert!(result.is_ok());
        let tokens = result.unwrap();
        assert!(tokens.is_empty());
    }

    // 测试57: 全部标点紧凑串
    #[test]
    fn test_all_punctuations_dense() {
        let input = "(){}[];,";
        let result = tokenize(input);
        assert!(result.is_ok());
        let tokens = result.unwrap();
        let output: Vec<String> = tokens.iter().map(|(t, l)| token_to_string(t, *l)).collect();
        assert_eq!(output, vec![
            "L_PAREN ( at Line 1.".to_string(),
            "R_PAREN ) at Line 1.".to_string(),
            "L_BRACE { at Line 1.".to_string(),
            "R_BRACE } at Line 1.".to_string(),
            "L_BRACKT [ at Line 1.".to_string(),
            "R_BRACKT ] at Line 1.".to_string(),
            "SEMICOLON ; at Line 1.".to_string(),
            "COMMA , at Line 1.".to_string(),
        ]);
    }

    // 测试58: 数字后紧跟括号与下标
    #[test]
    fn test_number_followed_by_paren_and_bracket() {
        let input = "42(arr[0]);";
        let result = tokenize(input);
        assert!(result.is_ok());
        let tokens = result.unwrap();
        let output: Vec<String> = tokens.iter().map(|(t, l)| token_to_string(t, *l)).collect();
        assert_eq!(output, vec![
            "INTEGER_CONST 42 at Line 1.".to_string(),
            "L_PAREN ( at Line 1.".to_string(),
            "IDENT arr at Line 1.".to_string(),
            "L_BRACKT [ at Line 1.".to_string(),
            "INTEGER_CONST 0 at Line 1.".to_string(),
            "R_BRACKT ] at Line 1.".to_string(),
            "R_PAREN ) at Line 1.".to_string(),
            "SEMICOLON ; at Line 1.".to_string(),
        ]);
    }

    // 测试59: 深度嵌套下标与混合进制常量
    #[test]
    fn test_deep_nested_index_with_mixed_bases() {
        let input = "a[b[c[0x1]+d[e[02]]]];";
        let result = tokenize(input);
        assert!(result.is_ok());
        let tokens = result.unwrap();
        let output: Vec<String> = tokens.iter().map(|(t, l)| token_to_string(t, *l)).collect();
        assert_eq!(output, vec![
            "IDENT a at Line 1.".to_string(),
            "L_BRACKT [ at Line 1.".to_string(),
            "IDENT b at Line 1.".to_string(),
            "L_BRACKT [ at Line 1.".to_string(),
            "IDENT c at Line 1.".to_string(),
            "L_BRACKT [ at Line 1.".to_string(),
            "INTEGER_CONST 1 at Line 1.".to_string(),
            "R_BRACKT ] at Line 1.".to_string(),
            "PLUS + at Line 1.".to_string(),
            "IDENT d at Line 1.".to_string(),
            "L_BRACKT [ at Line 1.".to_string(),
            "IDENT e at Line 1.".to_string(),
            "L_BRACKT [ at Line 1.".to_string(),
            "INTEGER_CONST 2 at Line 1.".to_string(),
            "R_BRACKT ] at Line 1.".to_string(),
            "R_BRACKT ] at Line 1.".to_string(),
            "R_BRACKT ] at Line 1.".to_string(),
            "R_BRACKT ] at Line 1.".to_string(),
            "SEMICOLON ; at Line 1.".to_string(),
        ]);
    }

    // 测试60: 0后跟字母与多0后跟标识符的拆分
    #[test]
    fn test_zero_followed_by_letters_splits() {
        let input = "0abc 000xyz";
        let result = tokenize(input);
        assert!(result.is_ok());
        let tokens = result.unwrap();
        let output: Vec<String> = tokens.iter().map(|(t, l)| token_to_string(t, *l)).collect();
        assert_eq!(output, vec![
            "INTEGER_CONST 0 at Line 1.".to_string(),
            "IDENT abc at Line 1.".to_string(),
            "INTEGER_CONST 0 at Line 1.".to_string(),
            "IDENT xyz at Line 1.".to_string(),
        ]);
    }

    // 测试61: 赋值链
    #[test]
    fn test_chained_assignments() {
        let input = "a=b=c=d=0;";
        let result = tokenize(input);
        assert!(result.is_ok());
        let tokens = result.unwrap();
        let output: Vec<String> = tokens.iter().map(|(t, l)| token_to_string(t, *l)).collect();
        assert_eq!(output, vec![
            "IDENT a at Line 1.".to_string(),
            "ASSIGN = at Line 1.".to_string(),
            "IDENT b at Line 1.".to_string(),
            "ASSIGN = at Line 1.".to_string(),
            "IDENT c at Line 1.".to_string(),
            "ASSIGN = at Line 1.".to_string(),
            "IDENT d at Line 1.".to_string(),
            "ASSIGN = at Line 1.".to_string(),
            "INTEGER_CONST 0 at Line 1.".to_string(),
            "SEMICOLON ; at Line 1.".to_string(),
        ]);
    }

    // 测试62: 带括号的return表达式
    #[test]
    fn test_return_with_parenthesized_expression() {
        let input = "return(x+y);";
        let result = tokenize(input);
        assert!(result.is_ok());
        let tokens = result.unwrap();
        let output: Vec<String> = tokens.iter().map(|(t, l)| token_to_string(t, *l)).collect();
        assert_eq!(output, vec![
            "RETURN return at Line 1.".to_string(),
            "L_PAREN ( at Line 1.".to_string(),
            "IDENT x at Line 1.".to_string(),
            "PLUS + at Line 1.".to_string(),
            "IDENT y at Line 1.".to_string(),
            "R_PAREN ) at Line 1.".to_string(),
            "SEMICOLON ; at Line 1.".to_string(),
        ]);
    }

    // 测试63: 多个非法字符但仅定位第一个错误行
    #[test]
    fn test_multiple_invalid_characters_first_error_line() {
        let input = "int a=1;\nint b=@#;\nint c=3;";
        let result = tokenize(input);
        assert!(result.is_err());
        let (msg, line_no) = result.unwrap_err();
        assert!(msg.contains("Mysterious character"));
        assert_eq!(line_no, 2);
    }

    // 测试64: 数字后下划线开头标识符的拆分
    #[test]
    fn test_number_then_underscore_identifier() {
        let input = "123__x";
        let result = tokenize(input);
        assert!(result.is_ok());
        let tokens = result.unwrap();
        let output: Vec<String> = tokens.iter().map(|(t, l)| token_to_string(t, *l)).collect();
        assert_eq!(output, vec![
            "INTEGER_CONST 123 at Line 1.".to_string(),
            "IDENT __x at Line 1.".to_string(),
        ]);
    }

    // 测试65: 标识符中包含 0xFF 字样不应被当作十六进制
    #[test]
    fn test_identifier_contains_hex_like_substring() {
        let input = "int var_0xFF=10;";
        let result = tokenize(input);
        assert!(result.is_ok());
        let tokens = result.unwrap();
        let output: Vec<String> = tokens.iter().map(|(t, l)| token_to_string(t, *l)).collect();
        assert_eq!(output, vec![
            "INT int at Line 1.".to_string(),
            "IDENT var_0xFF at Line 1.".to_string(),
            "ASSIGN = at Line 1.".to_string(),
            "INTEGER_CONST 10 at Line 1.".to_string(),
            "SEMICOLON ; at Line 1.".to_string(),
        ]);
    }

    // 测试66: 文件以 BOM (U+FEFF) 开头，应报错定位第1行
    #[test]
    fn test_file_starts_with_bom_error() {
        let input = "\u{FEFF}int x=1;";
        let result = tokenize(input);
        assert!(result.is_err());
        let (_msg, line_no) = result.unwrap_err();
        assert_eq!(line_no, 1);
    }

    // 测试67: 包含未被 WS 规则覆盖的垂直制表符，应报错
    #[test]
    fn test_vertical_tab_error() {
        let input = "int\x0B x=1;"; // \x0B 不在 [ \r\n\t]
        let result = tokenize(input);
        assert!(result.is_err());
        let (_msg, line_no) = result.unwrap_err();
        assert_eq!(line_no, 1);
    }

    // 测试68: 包含不间断空格 (U+00A0) ，应报错
    #[test]
    fn test_nbsp_error() {
        let input = "int\u{00A0}x=1;";
        let result = tokenize(input);
        assert!(result.is_err());
        let (_msg, line_no) = result.unwrap_err();
        assert_eq!(line_no, 1);
    }

    // 测试69: 行续接反斜杠（不支持），应在该行报错
    #[test]
    fn test_backslash_line_continuation_error() {
        let input = "int a = 1;\\\nint b = 2;";
        let result = tokenize(input);
        assert!(result.is_err());
        let (_msg, line_no) = result.unwrap_err();
        assert_eq!(line_no, 1);
    }

    // 测试70: 十进制超范围（超过 i64），应报错
    #[test]
    fn test_decimal_overflow_error() {
        let input = "int x = 9999999999999999999999999999;";
        let result = tokenize(input);
        assert!(result.is_err());
        let (msg, line_no) = result.unwrap_err();
        assert!(msg.contains("Invalid decimal number"));
        assert_eq!(line_no, 1);
    }

    // 测试71: 十六进制大小写混合且合法
    #[test]
    fn test_hex_mixed_case_legal() {
        let input = "int x=0xAbCd; int y=0XfF;";
        let result = tokenize(input);
        assert!(result.is_ok());
        let tokens = result.unwrap();
        let output: Vec<String> = tokens.iter().map(|(t, l)| token_to_string(t, *l)).collect();
        assert_eq!(output, vec![
            "INT int at Line 1.".to_string(),
            "IDENT x at Line 1.".to_string(),
            "ASSIGN = at Line 1.".to_string(),
            "INTEGER_CONST 43981 at Line 1.".to_string(),
            "SEMICOLON ; at Line 1.".to_string(),
            "INT int at Line 1.".to_string(),
            "IDENT y at Line 1.".to_string(),
            "ASSIGN = at Line 1.".to_string(),
            "INTEGER_CONST 255 at Line 1.".to_string(),
            "SEMICOLON ; at Line 1.".to_string(),
        ]);
    }

    // 测试72: 注释嵌在数字中间 0/*c*/1 -> 两个整数常量
    #[test]
    fn test_comment_inside_number_sequence() {
        let input = "0/*c*/1";
        let result = tokenize(input);
        assert!(result.is_ok());
        let tokens = result.unwrap();
        let output: Vec<String> = tokens.iter().map(|(t, l)| token_to_string(t, *l)).collect();
        assert_eq!(output, vec![
            "INTEGER_CONST 0 at Line 1.".to_string(),
            "INTEGER_CONST 1 at Line 1.".to_string(),
        ]);
    }

    // 测试73: 多个空注释相邻
    #[test]
    fn test_multiple_empty_comments_adjacent() {
        let input = "/**//* */ /*\n*/ /**/ int x=0;";
        let result = tokenize(input);
        assert!(result.is_ok());
        let tokens = result.unwrap();
        let output: Vec<String> = tokens.iter().map(|(t, l)| token_to_string(t, *l)).collect();
        assert_eq!(output, vec![
            "INT int at Line 2.".to_string(),
            "IDENT x at Line 2.".to_string(),
            "ASSIGN = at Line 2.".to_string(),
            "INTEGER_CONST 0 at Line 2.".to_string(),
            "SEMICOLON ; at Line 2.".to_string(),
        ]);
    }

    // 测试74: 不同类型注释相邻与无空白分隔
    #[test]
    fn test_adjacent_line_and_block_comments() {
        let input = "//a\n/*b*/int y=1;//c";
        let result = tokenize(input);
        assert!(result.is_ok());
        let tokens = result.unwrap();
        let output: Vec<String> = tokens.iter().map(|(t, l)| token_to_string(t, *l)).collect();
        assert_eq!(output, vec![
            "INT int at Line 2.".to_string(),
            "IDENT y at Line 2.".to_string(),
            "ASSIGN = at Line 2.".to_string(),
            "INTEGER_CONST 1 at Line 2.".to_string(),
            "SEMICOLON ; at Line 2.".to_string(),
        ]);
    }

    // 测试75: 仅标点与空白的行
    #[test]
    fn test_only_punctuations_and_whitespace_lines() {
        let input = "  ; \t , \n ;";
        let result = tokenize(input);
        assert!(result.is_ok());
        let tokens = result.unwrap();
        let output: Vec<String> = tokens.iter().map(|(t, l)| token_to_string(t, *l)).collect();
        assert_eq!(output, vec![
            "SEMICOLON ; at Line 1.".to_string(),
            "COMMA , at Line 1.".to_string(),
            "SEMICOLON ; at Line 2.".to_string(),
        ]);
    }

    // 测试76: 仅CR换行分隔的两行
    #[test]
    fn test_carriage_return_only_newlines() {
        let input = "int a=1;\rint b=2;";
        let result = tokenize(input);
        assert!(result.is_ok());
        let tokens = result.unwrap();
        let out: Vec<String> = tokens.iter().map(|(t,l)| token_to_string(t, *l)).collect();
        assert_eq!(out[0], "INT int at Line 1.");
        assert_eq!(out[1], "IDENT a at Line 1.");
        assert_eq!(out[2], "ASSIGN = at Line 1.");
        assert_eq!(out[3], "INTEGER_CONST 1 at Line 1.");
        assert_eq!(out[4], "SEMICOLON ; at Line 1.");
        // 第二行
        assert!(out.iter().any(|s| s == "INT int at Line 2."));
    }

    // 测试77: Form Feed (\x0C) 非法空白，应报错
    #[test]
    fn test_form_feed_error() {
        let input = "int\x0Cx=0;";
        let result = tokenize(input);
        assert!(result.is_err());
        let (_msg, line_no) = result.unwrap_err();
        assert_eq!(line_no, 1);
    }

    // 测试78: 三斜杠注释（按行注释处理）
    #[test]
    fn test_triple_slash_comment() {
        let input = "int x=1; /// doc\nint y=2;";
        let result = tokenize(input);
        assert!(result.is_ok());
        let tokens = result.unwrap();
        let out: Vec<String> = tokens.iter().map(|(t,l)| token_to_string(t, *l)).collect();
        assert_eq!(out, vec![
            "INT int at Line 1.".to_string(),
            "IDENT x at Line 1.".to_string(),
            "ASSIGN = at Line 1.".to_string(),
            "INTEGER_CONST 1 at Line 1.".to_string(),
            "SEMICOLON ; at Line 1.".to_string(),
            "INT int at Line 2.".to_string(),
            "IDENT y at Line 2.".to_string(),
            "ASSIGN = at Line 2.".to_string(),
            "INTEGER_CONST 2 at Line 2.".to_string(),
            "SEMICOLON ; at Line 2.".to_string(),
        ]);
    }

    // 测试79: 极短未闭合块注释 "/*/" 应报错行1
    #[test]
    fn test_minimal_unclosed_block_comment() {
        let input = "/*/";
        let result = tokenize(input);
        assert!(result.is_err());
        let (_msg, line_no) = result.unwrap_err();
        assert_eq!(line_no, 1);
    }

    // 测试80: 十六进制后紧跟字母（非十六进制）拆分
    #[test]
    fn test_hex_then_letter_split() {
        let input = "0x1z";
        let result = tokenize(input);
        assert!(result.is_ok());
        let tokens = result.unwrap();
        let out: Vec<String> = tokens.iter().map(|(t,l)| token_to_string(t, *l)).collect();
        assert_eq!(out, vec![
            "INTEGER_CONST 1 at Line 1.".to_string(),
            "IDENT z at Line 1.".to_string(),
        ]);
    }

    // 测试81: 多个前导零后跟十进制与标识符
    #[test]
    fn test_many_leading_zeros_then_ident() {
        let input = "0000009abc";
        let result = tokenize(input);
        assert!(result.is_ok());
        let tokens = result.unwrap();
        let out: Vec<String> = tokens.iter().map(|(t,l)| token_to_string(t, *l)).collect();
        assert_eq!(out, vec![
            "INTEGER_CONST 0 at Line 1.".to_string(),
            "INTEGER_CONST 0 at Line 1.".to_string(),
            "INTEGER_CONST 0 at Line 1.".to_string(),
            "INTEGER_CONST 0 at Line 1.".to_string(),
            "INTEGER_CONST 0 at Line 1.".to_string(),
            "INTEGER_CONST 0 at Line 1.".to_string(),
            "INTEGER_CONST 9 at Line 1.".to_string(),
            "IDENT abc at Line 1.".to_string(),
        ]);
    }

    // 测试82: 密集操作符链
    #[test]
    fn test_dense_operator_chain() {
        let input = "!!==&&||";
        let result = tokenize(input);
        assert!(result.is_ok());
        let tokens = result.unwrap();
        let out: Vec<String> = tokens.iter().map(|(t,l)| token_to_string(t, *l)).collect();
        assert_eq!(out, vec![
            "NOT ! at Line 1.".to_string(),
            "NOT ! at Line 1.".to_string(),
            "EQ == at Line 1.".to_string(),
            "AND && at Line 1.".to_string(),
            "OR || at Line 1.".to_string(),
        ]);
    }

    // 测试83: 操作符两侧夹注释
    #[test]
    fn test_operators_with_comments_between() {
        let input = "a/*c*/<=/*d*/=b";
        let result = tokenize(input);
        assert!(result.is_ok());
        let tokens = result.unwrap();
        let out: Vec<String> = tokens.iter().map(|(t,l)| token_to_string(t, *l)).collect();
        assert_eq!(out, vec![
            "IDENT a at Line 1.".to_string(),
            "LE <= at Line 1.".to_string(),
            "ASSIGN = at Line 1.".to_string(),
            "IDENT b at Line 1.".to_string(),
        ]);
    }

    // 测试84: CRLF 与空行的行号正确性
    #[test]
    fn test_crlf_and_blank_lines_line_numbers() {
        let input = "int a=1;\r\n\r\nint b=2;";
        let result = tokenize(input);
        assert!(result.is_ok());
        let tokens = result.unwrap();
        let out: Vec<String> = tokens.iter().map(|(t,l)| token_to_string(t, *l)).collect();
        assert!(out.iter().any(|s| s == "INT int at Line 1."));
        assert!(out.iter().any(|s| s == "INT int at Line 3."));
    }

    // 测试85: 超长标识符与夹杂数字
    #[test]
    fn test_very_long_identifier_with_digits() {
        let long = "x".to_string() + &"_".repeat(200) + "123";
        let input = format!("int {}=0;", long);
        let result = tokenize(&input);
        assert!(result.is_ok());
        let tokens = result.unwrap();
        let out: Vec<String> = tokens.iter().map(|(t,l)| token_to_string(t, *l)).collect();
        assert_eq!(out[0], "INT int at Line 1.");
        assert_eq!(out[1], format!("IDENT {} at Line 1.", long));
        assert_eq!(out[2], "ASSIGN = at Line 1.");
        assert_eq!(out[3], "INTEGER_CONST 0 at Line 1.");
        assert_eq!(out[4], "SEMICOLON ; at Line 1.");
    }

    // 测试86: 十六进制 0x01 测试用例
    #[test]
    fn test_hex_0x01() {
        let input = "0x01";
        let result = tokenize(input);
        assert!(result.is_ok());
        let tokens = result.unwrap();
        let output: Vec<String> = tokens.iter()
            .map(|(token, line_no)| token_to_string(token, *line_no))
            .collect();
        
        assert_eq!(output, vec![
            "INTEGER_CONST 0 at Line 1.".to_string(),
            "INTEGER_CONST 1 at Line 1.".to_string(),
        ]);
    }

    // 测试87: 八进制 08 测试用例（应该拆分为 0 和 8）
    #[test]
    fn test_octal_08_split() {
        let input = "08";
        let result = tokenize(input);
        assert!(result.is_ok());
        let tokens = result.unwrap();
        let output: Vec<String> = tokens.iter()
            .map(|(token, line_no)| token_to_string(token, *line_no))
            .collect();
        
        assert_eq!(output, vec![
            "INTEGER_CONST 0 at Line 1.".to_string(),
            "INTEGER_CONST 8 at Line 1.".to_string(),
        ]);
    }
}